2024-09-21T09:32:39.523 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-21T09:32:39.552 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T09:32:39.642 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658 branch: main description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} email: ceph-qa@ceph.com first_in_suite: false job_id: '7913658' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run run randomized correctness test for rados operations on an erasure-coded pool ' - desc: 'object class functional tests ' - desc: 'generate read/write load with rados objects ranging from 1MB to 25MB ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'librbd python api tests ' name: teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(POOL_APP_NOT_ENABLED\) - OSD_DOWN - mons down - mon down - MON_DOWN - out of quorum - PG_DEGRADED - Reduced data availability - Degraded data redundancy - OSDMAP_FLAGS - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: extra_system_packages: - python3-pytest flavor: default sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - mds.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mds.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 5357 sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 sleep_before_teardown: 0 subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 targets: smithi033.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHa6vMKbtLccdkPQEFHWunUDYe6kOQiNLiOTwcN2b7QdvnQE1gFxnE6f9JVstb5SNsl3oXQueBeFDZlVTwd6Hdg= smithi105.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYXCy6tK5ExN6gJRT/kcFXHoPKSI+Z87s1wCGdB2Jp3PPATP+Ffb458Wq6Mrzh+WNljEPvHdd9ZRrzaF1x3Pjo= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - print: '**** done install task...' - print: '**** done start installing quincy cephadm ...' - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - print: '**** done end installing quincy cephadm ...' - print: '**** done start cephadm.shell ceph config set mgr...' - cephadm.shell: mon.a: - ceph config set mgr mgr/cephadm/use_repo_digest true --force - print: '**** done cephadm.shell ceph config set mgr...' - print: '**** done start telemetry quincy...' - workunit: clients: client.0: - test_telemetry_quincy.sh - print: '**** done end telemetry quincy...' - print: '**** done start parallel' - parallel: - workload - upgrade-sequence - print: '**** done end parallel' - print: '**** done start telemetry x...' - workunit: clients: client.0: - test_telemetry_quincy_x.sh - print: '**** done end telemetry x...' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-21_03:08:26 tube: smithi upgrade-sequence: sequential: - print: '**** done start upgrade, wait' - cephadm.shell: env: - sha1 mon.a: - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - print: '**** done end upgrade, wait...' user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2902 workload: full_sequential: - print: '**** done start ec-rados-default.yaml' - rados: clients: - client.0 ec_pool: true objects: 50 op_weights: append: 100 copy_from: 50 delete: 50 read: 100 rmattr: 25 rollback: 50 setattr: 25 snap_create: 50 snap_remove: 50 write: 0 ops: 4000 write_append_excl: false - print: '**** done end ec-rados-default.yaml' - print: '**** done start rados_api.yaml' - workunit: branch: quincy clients: client.0: - cls - print: '**** done end rados_api.yaml' - print: '**** done start rados_loadgenbig.yaml' - workunit: branch: quincy clients: client.0: - rados/load-gen-big.sh - print: '**** done end rados_loadgenbig.yaml' - print: '**** done start rbd_import_export.yaml' - workunit: branch: quincy clients: client.1: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done end rbd_import_export.yaml' - print: '**** done start test_rbd_api.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_api.yaml' - print: '**** done start test_rbd_python.yaml' - workunit: branch: quincy clients: client.0: - rbd/test_librbd_python.sh env: RBD_FEATURES: '61' - print: '**** done end test_rbd_python.yaml' 2024-09-21T09:32:39.642 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa; will attempt to use it 2024-09-21T09:32:39.643 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks 2024-09-21T09:32:39.643 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-21T09:32:39.644 INFO:teuthology.task.internal:Checking packages... 2024-09-21T09:32:39.665 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '62da54b4b83a96ca9976b7e1d0c8272a564a2208' 2024-09-21T09:32:39.665 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:32:39.665 INFO:teuthology.packaging:ref: None 2024-09-21T09:32:39.665 INFO:teuthology.packaging:tag: None 2024-09-21T09:32:39.665 INFO:teuthology.packaging:branch: main 2024-09-21T09:32:39.665 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:32:39.665 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-21T09:32:39.911 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-5092.g62da54b4 2024-09-21T09:32:39.913 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-21T09:32:39.919 INFO:teuthology.task.internal:no buildpackages task found 2024-09-21T09:32:39.919 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-21T09:32:39.928 INFO:teuthology.task.internal:Saving configuration 2024-09-21T09:32:39.947 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-21T09:32:39.954 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-21T09:32:39.980 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi033.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:27:43.251681', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHa6vMKbtLccdkPQEFHWunUDYe6kOQiNLiOTwcN2b7QdvnQE1gFxnE6f9JVstb5SNsl3oXQueBeFDZlVTwd6Hdg='} 2024-09-21T09:32:40.001 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi105.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-21 09:27:43.249783', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLYXCy6tK5ExN6gJRT/kcFXHoPKSI+Z87s1wCGdB2Jp3PPATP+Ffb458Wq6Mrzh+WNljEPvHdd9ZRrzaF1x3Pjo='} 2024-09-21T09:32:40.001 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-21T09:32:40.007 INFO:teuthology.task.internal:roles: ubuntu@smithi033.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-21T09:32:40.007 INFO:teuthology.task.internal:roles: ubuntu@smithi105.front.sepia.ceph.com - ['mon.b', 'mds.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-21T09:32:40.007 INFO:teuthology.run_tasks:Running task console_log... 2024-09-21T09:32:40.058 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7faa5d4b0160>, signals=[15]) 2024-09-21T09:32:40.058 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-21T09:32:40.065 INFO:teuthology.task.internal:Opening connections... 2024-09-21T09:32:40.065 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi033.front.sepia.ceph.com 2024-09-21T09:32:40.067 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:32:40.167 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T09:32:40.167 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:32:40.248 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-21T09:32:40.254 DEBUG:teuthology.orchestra.run.smithi033:> uname -m 2024-09-21T09:32:40.273 INFO:teuthology.orchestra.run.smithi033.stdout:x86_64 2024-09-21T09:32:40.273 DEBUG:teuthology.orchestra.run.smithi033:> cat /etc/os-release 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:NAME="CentOS Stream" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:VERSION="9" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:ID="centos" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:VERSION_ID="9" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:ANSI_COLOR="0;31" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:32:40.331 INFO:teuthology.orchestra.run.smithi033.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:32:40.332 INFO:teuthology.orchestra.run.smithi033.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:32:40.332 INFO:teuthology.orchestra.run.smithi033.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:32:40.332 INFO:teuthology.orchestra.run.smithi033.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:32:40.332 INFO:teuthology.lock.ops:Updating smithi033.front.sepia.ceph.com on lock server 2024-09-21T09:32:40.353 DEBUG:teuthology.orchestra.run.smithi105:> uname -m 2024-09-21T09:32:40.372 INFO:teuthology.orchestra.run.smithi105.stdout:x86_64 2024-09-21T09:32:40.372 DEBUG:teuthology.orchestra.run.smithi105:> cat /etc/os-release 2024-09-21T09:32:40.429 INFO:teuthology.orchestra.run.smithi105.stdout:NAME="CentOS Stream" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION="9" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:ID="centos" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:ID_LIKE="rhel fedora" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:VERSION_ID="9" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:PLATFORM_ID="platform:el9" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:ANSI_COLOR="0;31" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:LOGO="fedora-logo-icon" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:HOME_URL="https://centos.org/" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-21T09:32:40.430 INFO:teuthology.orchestra.run.smithi105.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-21T09:32:40.431 INFO:teuthology.lock.ops:Updating smithi105.front.sepia.ceph.com on lock server 2024-09-21T09:32:40.453 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-21T09:32:40.463 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-21T09:32:40.469 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-21T09:32:40.469 DEBUG:teuthology.orchestra.run.smithi033:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:32:40.472 DEBUG:teuthology.orchestra.run.smithi105:> test '!' -e /home/ubuntu/cephtest 2024-09-21T09:32:40.489 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-21T09:32:40.494 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-21T09:32:40.494 DEBUG:teuthology.orchestra.run.smithi033:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:32:40.529 DEBUG:teuthology.orchestra.run.smithi105:> test -z $(ls -A /var/lib/ceph) 2024-09-21T09:32:40.619 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-21T09:32:40.681 INFO:teuthology.run_tasks:Running task kernel... 2024-09-21T09:32:40.696 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-21T09:32:40.696 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-21T09:32:40.696 DEBUG:teuthology.orchestra.run.smithi033:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:32:40.697 DEBUG:teuthology.orchestra.run.smithi105:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-21T09:32:40.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:32:40.714 DEBUG:teuthology.orchestra.run.smithi033:> uname -r 2024-09-21T09:32:40.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:32:40.715 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-21T09:32:40.772 INFO:teuthology.orchestra.run.smithi033.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:32:40.772 INFO:teuthology.task.kernel:Running kernel on smithi033: 5.14.0-503.el9.x86_64 2024-09-21T09:32:40.772 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum install -y kernel 2024-09-21T09:32:40.774 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-503.el9.x86_64 2024-09-21T09:32:40.774 INFO:teuthology.task.kernel:Running kernel on smithi105: 5.14.0-503.el9.x86_64 2024-09-21T09:32:40.774 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum install -y kernel 2024-09-21T09:32:42.913 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.3 MB 00:00 2024-09-21T09:32:44.273 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - BaseOS 4.9 MB/s | 8.3 MB 00:01 2024-09-21T09:32:44.648 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - AppStream 31 MB/s | 20 MB 00:00 2024-09-21T09:32:46.091 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - AppStream 29 MB/s | 20 MB 00:00 2024-09-21T09:32:50.082 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - CRB 4.0 MB/s | 6.5 MB 00:01 2024-09-21T09:32:50.567 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.5 MB 00:00 2024-09-21T09:32:51.507 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - Extras packages 94 kB/s | 19 kB 00:00 2024-09-21T09:32:52.076 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - Extras packages 86 kB/s | 19 kB 00:00 2024-09-21T09:32:52.278 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 35 MB/s | 23 MB 00:00 2024-09-21T09:32:52.871 INFO:teuthology.orchestra.run.smithi033.stdout:Extra Packages for Enterprise Linux 34 MB/s | 23 MB 00:00 2024-09-21T09:32:57.643 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 33 kB/s | 1.7 kB 00:00 2024-09-21T09:32:58.444 INFO:teuthology.orchestra.run.smithi033.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-21T09:32:59.565 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:32:59.566 INFO:teuthology.orchestra.run.smithi105.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:32:59.679 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:32:59.686 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:32:59.687 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:32:59.687 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T09:32:59.687 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:32:59.687 INFO:teuthology.orchestra.run.smithi105.stdout:Install 4 Packages 2024-09-21T09:32:59.687 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:32:59.687 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 79 M 2024-09-21T09:32:59.687 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 126 M 2024-09-21T09:32:59.687 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T09:32:59.794 INFO:teuthology.orchestra.run.smithi105.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 418 kB/s | 36 kB 00:00 2024-09-21T09:33:00.178 INFO:teuthology.orchestra.run.smithi105.stdout:(2/4): kernel-core-5.14.0-508.el9.x86_64.rpm 33 MB/s | 16 MB 00:00 2024-09-21T09:33:00.417 INFO:teuthology.orchestra.run.smithi033.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-21T09:33:00.417 INFO:teuthology.orchestra.run.smithi033.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-21T09:33:00.511 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-09-21T09:33:00.517 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:33:00.517 INFO:teuthology.orchestra.run.smithi033.stdout: Package Architecture Version Repository Size 2024-09-21T09:33:00.517 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:33:00.517 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout: kernel x86_64 5.14.0-508.el9 baseos 36 k 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-core x86_64 5.14.0-508.el9 baseos 16 M 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules x86_64 5.14.0-508.el9 baseos 35 M 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-core x86_64 5.14.0-508.el9 baseos 29 M 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout:Install 4 Packages 2024-09-21T09:33:00.518 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:33:00.519 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 79 M 2024-09-21T09:33:00.519 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 126 M 2024-09-21T09:33:00.519 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-09-21T09:33:00.619 INFO:teuthology.orchestra.run.smithi105.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 38 MB/s | 35 MB 00:00 2024-09-21T09:33:00.761 INFO:teuthology.orchestra.run.smithi033.stdout:(1/4): kernel-5.14.0-508.el9.x86_64.rpm 309 kB/s | 36 kB 00:00 2024-09-21T09:33:00.970 INFO:teuthology.orchestra.run.smithi105.stdout:(4/4): kernel-modules-core-5.14.0-508.el9.x86_6 24 MB/s | 29 MB 00:01 2024-09-21T09:33:00.970 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:33:00.970 INFO:teuthology.orchestra.run.smithi105.stdout:Total 61 MB/s | 79 MB 00:01 2024-09-21T09:33:01.648 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T09:33:02.018 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T09:33:02.018 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T09:33:02.954 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T09:33:02.955 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T09:33:03.778 INFO:teuthology.orchestra.run.smithi033.stdout:(2/4): kernel-modules-core-5.14.0-508.el9.x86_6 9.5 MB/s | 29 MB 00:03 2024-09-21T09:33:05.395 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T09:33:06.107 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:33:06.203 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:33:06.403 INFO:teuthology.orchestra.run.smithi033.stdout:(3/4): kernel-modules-5.14.0-508.el9.x86_64.rpm 6.0 MB/s | 35 MB 00:05 2024-09-21T09:33:07.311 INFO:teuthology.orchestra.run.smithi033.stdout:(4/4): kernel-core-5.14.0-508.el9.x86_64.rpm 2.3 MB/s | 16 MB 00:06 2024-09-21T09:33:07.312 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:33:07.312 INFO:teuthology.orchestra.run.smithi033.stdout:Total 12 MB/s | 79 MB 00:06 2024-09-21T09:33:08.019 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-09-21T09:33:08.392 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-09-21T09:33:08.393 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-09-21T09:33:08.971 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:33:09.115 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:33:09.389 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-09-21T09:33:09.389 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-09-21T09:33:12.044 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-09-21T09:33:12.713 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-modules-core-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:33:12.795 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:33:13.997 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:33:14.032 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:15.551 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:33:15.683 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:33:19.079 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:20.809 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:33:20.850 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:25.980 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:57.427 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:57.476 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:57.859 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:57.859 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:33:57.859 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:33:57.859 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:33:58.703 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:33:58.704 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:33:58.704 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T09:33:58.704 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:33:58.704 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:33:58.704 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:33:58.704 INFO:teuthology.orchestra.run.smithi105.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:33:58.704 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:33:58.704 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T09:33:59.059 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:33:59.140 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:33:59.141 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:33:59.141 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:33:59.141 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:33:59.141 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:33:59.141 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-21T09:33:59.141 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi105.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:33:59.141 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:33:59.200 INFO:teuthology.orchestra.run.smithi105.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:33:59.200 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:33:59.201 DEBUG:teuthology.orchestra.run.smithi105:> sudo rpm -qi grub2-tools 2024-09-21T09:33:59.232 INFO:teuthology.orchestra.run.smithi105.stdout:Name : grub2-tools 2024-09-21T09:33:59.232 INFO:teuthology.orchestra.run.smithi105.stdout:Epoch : 1 2024-09-21T09:33:59.232 INFO:teuthology.orchestra.run.smithi105.stdout:Version : 2.06 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Release : 82.el9 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Architecture: x86_64 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Group : Unspecified 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Size : 8274670 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:License : GPLv3+ 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Packager : builder@centos.org 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Vendor : CentOS 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Summary : Support tools for GRUB. 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout:Description : 2024-09-21T09:33:59.233 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:33:59.234 INFO:teuthology.orchestra.run.smithi105.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:33:59.234 INFO:teuthology.orchestra.run.smithi105.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:33:59.234 INFO:teuthology.orchestra.run.smithi105.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:33:59.234 INFO:teuthology.orchestra.run.smithi105.stdout:hardware devices. 2024-09-21T09:33:59.234 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:33:59.234 INFO:teuthology.orchestra.run.smithi105.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:33:59.235 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:33:59.235 INFO:teuthology.task.kernel:Updating grub on smithi105 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:33:59.235 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:33:59.985 INFO:teuthology.orchestra.run.smithi105.stderr:Generating grub configuration file ... 2024-09-21T09:34:01.326 INFO:teuthology.orchestra.run.smithi105.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:34:01.359 INFO:teuthology.orchestra.run.smithi105.stderr:done 2024-09-21T09:34:01.361 DEBUG:teuthology.orchestra.run.smithi105:> mktemp 2024-09-21T09:34:01.417 INFO:teuthology.orchestra.run.smithi105.stdout:/tmp/tmp.P8HxyavbBk 2024-09-21T09:34:01.418 DEBUG:teuthology.orchestra.run.smithi105:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.P8HxyavbBk 2024-09-21T09:34:01.494 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0666 /tmp/tmp.P8HxyavbBk 2024-09-21T09:34:01.643 DEBUG:teuthology.orchestra.remote:smithi105:/tmp/tmp.P8HxyavbBk is 6KB 2024-09-21T09:34:01.691 DEBUG:teuthology.orchestra.run.smithi105:> rm -fr /tmp/tmp.P8HxyavbBk 2024-09-21T09:34:01.706 DEBUG:teuthology.orchestra.run.smithi105:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:34:01.783 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:34:01.783 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:34:01.783 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:34:01.783 INFO:teuthology.orchestra.run.smithi105.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:34:01.785 DEBUG:teuthology.orchestra.run.smithi105:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:34:01.943 DEBUG:teuthology.orchestra.run.smithi105:> sudo shutdown -r now 2024-09-21T09:34:04.787 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:04.828 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-modules-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:05.190 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: kernel-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:05.190 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-5.14.0-508.el9.x86_64 1/4 2024-09-21T09:34:05.191 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-core-5.14.0-508.el9.x86_64 2/4 2024-09-21T09:34:05.191 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-modules-5.14.0-508.el9.x86_64 3/4 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : kernel-modules-core-5.14.0-508.el9.x86_64 4/4 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-5.14.0-508.el9.x86_64 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-core-5.14.0-508.el9.x86_64 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-5.14.0-508.el9.x86_64 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout: kernel-modules-core-5.14.0-508.el9.x86_64 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:34:06.115 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-09-21T09:34:06.466 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:34:06.593 INFO:teuthology.orchestra.run.smithi033.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:34:06.593 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:34:06.593 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-508.el9.x86_64 2024-09-21T09:34:06.593 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-21T09:34:06.593 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-21T09:34:06.593 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-21T09:34:06.594 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi033.front.sepia.ceph.com, path=None, version=distro) 2024-09-21T09:34:06.594 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q kernel | sort -rV | head -n 1 2024-09-21T09:34:06.652 INFO:teuthology.orchestra.run.smithi033.stdout:kernel-5.14.0-508.el9.x86_64 2024-09-21T09:34:06.652 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-508.el9.x86_64 2024-09-21T09:34:06.653 DEBUG:teuthology.orchestra.run.smithi033:> sudo rpm -qi grub2-tools 2024-09-21T09:34:06.684 INFO:teuthology.orchestra.run.smithi033.stdout:Name : grub2-tools 2024-09-21T09:34:06.684 INFO:teuthology.orchestra.run.smithi033.stdout:Epoch : 1 2024-09-21T09:34:06.684 INFO:teuthology.orchestra.run.smithi033.stdout:Version : 2.06 2024-09-21T09:34:06.684 INFO:teuthology.orchestra.run.smithi033.stdout:Release : 82.el9 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Architecture: x86_64 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Group : Unspecified 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Size : 8274670 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:License : GPLv3+ 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Packager : builder@centos.org 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Vendor : CentOS 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Summary : Support tools for GRUB. 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout:Description : 2024-09-21T09:34:06.685 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:34:06.686 INFO:teuthology.orchestra.run.smithi033.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-21T09:34:06.686 INFO:teuthology.orchestra.run.smithi033.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-21T09:34:06.686 INFO:teuthology.orchestra.run.smithi033.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-21T09:34:06.686 INFO:teuthology.orchestra.run.smithi033.stdout:hardware devices. 2024-09-21T09:34:06.686 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:34:06.686 INFO:teuthology.orchestra.run.smithi033.stdout:This subpackage provides tools for support of all platforms. 2024-09-21T09:34:06.687 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-21T09:34:06.687 INFO:teuthology.task.kernel:Updating grub on smithi033 to boot 5.14.0-508.el9.x86_64 2024-09-21T09:34:06.687 DEBUG:teuthology.orchestra.run.smithi033:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-21T09:34:07.233 INFO:teuthology.orchestra.run.smithi033.stderr:Generating grub configuration file ... 2024-09-21T09:34:08.627 INFO:teuthology.orchestra.run.smithi033.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-21T09:34:08.657 INFO:teuthology.orchestra.run.smithi033.stderr:done 2024-09-21T09:34:08.663 DEBUG:teuthology.orchestra.run.smithi033:> mktemp 2024-09-21T09:34:08.680 INFO:teuthology.orchestra.run.smithi033.stdout:/tmp/tmp.3gdcAwCXLF 2024-09-21T09:34:08.681 DEBUG:teuthology.orchestra.run.smithi033:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.3gdcAwCXLF 2024-09-21T09:34:08.757 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 0666 /tmp/tmp.3gdcAwCXLF 2024-09-21T09:34:08.902 DEBUG:teuthology.orchestra.remote:smithi033:/tmp/tmp.3gdcAwCXLF is 6KB 2024-09-21T09:34:08.923 DEBUG:teuthology.orchestra.run.smithi033:> rm -fr /tmp/tmp.3gdcAwCXLF 2024-09-21T09:34:08.937 DEBUG:teuthology.orchestra.run.smithi033:> sudo /bin/ls /boot/loader/entries || true 2024-09-21T09:34:09.008 INFO:teuthology.orchestra.run.smithi033.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-21T09:34:09.008 INFO:teuthology.orchestra.run.smithi033.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-21T09:34:09.008 INFO:teuthology.orchestra.run.smithi033.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-21T09:34:09.008 INFO:teuthology.orchestra.run.smithi033.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64.conf 2024-09-21T09:34:09.009 DEBUG:teuthology.orchestra.run.smithi033:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-508.el9.x86_64 2024-09-21T09:34:09.144 DEBUG:teuthology.orchestra.run.smithi033:> sudo shutdown -r now 2024-09-21T09:34:31.950 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-21T09:34:31.950 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T09:34:31.951 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:34:39.150 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-21T09:34:39.151 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-21T09:34:39.151 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:35:31.987 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T09:35:39.156 DEBUG:teuthology.orchestra.remote:timed out 2024-09-21T09:35:40.990 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T09:35:40.990 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:35:41.221 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-21T09:35:41.612 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T09:35:41.612 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:35:41.612 DEBUG:teuthology.orchestra.run.smithi105:> uname -r 2024-09-21T09:35:41.629 INFO:teuthology.orchestra.run.smithi105.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:35:41.629 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:35:41.629 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:35:41.629 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:35:42.630 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-21T09:35:42.631 DEBUG:teuthology.orchestra.run.smithi105:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:35:42.693 INFO:teuthology.orchestra.run.smithi105.stdout:ttyS1 2024-09-21T09:35:42.722 DEBUG:teuthology.parallel:result is None 2024-09-21T09:35:48.157 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-21T09:35:48.157 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:35:48.593 DEBUG:teuthology.orchestra.run.smithi033:> true 2024-09-21T09:35:48.955 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-21T09:35:48.956 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-508.el9.x86_64"... 2024-09-21T09:35:48.956 DEBUG:teuthology.orchestra.run.smithi033:> uname -r 2024-09-21T09:35:48.971 INFO:teuthology.orchestra.run.smithi033.stdout:5.14.0-508.el9.x86_64 2024-09-21T09:35:48.971 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-508.el9.x86_64 vs 5.14.0-508.el9.x86_64 2024-09-21T09:35:48.971 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-21T09:35:48.971 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-21T09:35:49.972 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-21T09:35:49.973 DEBUG:teuthology.orchestra.run.smithi033:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-21T09:35:50.033 INFO:teuthology.orchestra.run.smithi033.stdout:ttyS1 2024-09-21T09:35:50.058 DEBUG:teuthology.parallel:result is None 2024-09-21T09:35:50.058 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-21T09:35:50.070 INFO:teuthology.task.internal:Creating test directory... 2024-09-21T09:35:50.070 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:35:50.072 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-21T09:35:50.089 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-21T09:35:50.095 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-21T09:35:50.101 INFO:teuthology.task.internal:Creating archive directory... 2024-09-21T09:35:50.101 DEBUG:teuthology.orchestra.run.smithi033:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:35:50.128 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-21T09:35:50.160 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-21T09:35:50.166 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-21T09:35:50.167 DEBUG:teuthology.orchestra.run.smithi033:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:35:50.191 DEBUG:teuthology.orchestra.run.smithi105:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-21T09:35:50.217 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:35:50.227 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:35:50.242 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:35:50.251 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-21T09:35:50.253 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-21T09:35:50.260 INFO:teuthology.task.internal:Configuring sudo... 2024-09-21T09:35:50.260 DEBUG:teuthology.orchestra.run.smithi033:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:35:50.271 DEBUG:teuthology.orchestra.run.smithi105:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-21T09:35:50.296 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-21T09:35:50.304 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-21T09:35:50.305 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:35:50.337 DEBUG:teuthology.orchestra.run.smithi105:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-21T09:35:50.353 DEBUG:teuthology.orchestra.run.smithi033:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:35:50.434 DEBUG:teuthology.orchestra.run.smithi033:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:35:50.481 DEBUG:teuthology.orchestra.run.smithi033:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:35:50.562 DEBUG:teuthology.orchestra.run.smithi033:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:35:50.604 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:35:50.604 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:35:50.669 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:35:50.712 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-21T09:35:50.762 DEBUG:teuthology.orchestra.run.smithi105:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:35:50.843 DEBUG:teuthology.orchestra.run.smithi105:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-21T09:35:50.882 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:35:50.882 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-21T09:35:50.949 DEBUG:teuthology.orchestra.run.smithi033:> sudo service rsyslog restart 2024-09-21T09:35:50.951 DEBUG:teuthology.orchestra.run.smithi105:> sudo service rsyslog restart 2024-09-21T09:35:50.998 INFO:teuthology.orchestra.run.smithi033.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:35:51.044 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T09:35:51.312 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-21T09:35:51.319 INFO:teuthology.task.internal:Starting timer... 2024-09-21T09:35:51.319 INFO:teuthology.run_tasks:Running task pcp... 2024-09-21T09:35:51.329 INFO:teuthology.run_tasks:Running task selinux... 2024-09-21T09:35:51.337 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-21T09:35:51.337 DEBUG:teuthology.orchestra.run.smithi033:> sudo service auditd rotate 2024-09-21T09:35:51.387 INFO:teuthology.orchestra.run.smithi033.stdout:Rotating logs: 2024-09-21T09:35:51.388 DEBUG:teuthology.orchestra.run.smithi105:> sudo service auditd rotate 2024-09-21T09:35:51.471 INFO:teuthology.orchestra.run.smithi105.stdout:Rotating logs: 2024-09-21T09:35:51.473 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-21T09:35:51.473 DEBUG:teuthology.orchestra.run.smithi033:> /usr/sbin/getenforce 2024-09-21T09:35:51.517 INFO:teuthology.orchestra.run.smithi033.stdout:Permissive 2024-09-21T09:35:51.518 DEBUG:teuthology.orchestra.run.smithi105:> /usr/sbin/getenforce 2024-09-21T09:35:51.548 INFO:teuthology.orchestra.run.smithi105.stdout:Permissive 2024-09-21T09:35:51.863 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi033.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi105.front.sepia.ceph.com': 'permissive'} 2024-09-21T09:35:51.864 DEBUG:teuthology.orchestra.run.smithi033:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:35:51.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:35:51.891 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T09:35:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:35:51.919 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-21T09:35:51.919 DEBUG:teuthology.orchestra.run.smithi033:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:35:52.015 DEBUG:teuthology.orchestra.run.smithi105:> sudo /usr/sbin/setenforce permissive 2024-09-21T09:35:52.048 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-21T09:35:52.059 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-21T09:35:52.069 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-21T09:35:52.069 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi033.front.sepia.ceph.com,smithi105.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-21T09:39:35.931 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi033.front.sepia.ceph.com'), Remote(name='ubuntu@smithi105.front.sepia.ceph.com')] 2024-09-21T09:39:35.932 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-21T09:39:35.933 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi033.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:39:36.010 DEBUG:teuthology.orchestra.run.smithi033:> true 2024-09-21T09:39:36.086 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi033.front.sepia.ceph.com' 2024-09-21T09:39:36.086 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T09:39:36.087 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi105.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-21T09:39:36.161 DEBUG:teuthology.orchestra.run.smithi105:> true 2024-09-21T09:39:36.196 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi105.front.sepia.ceph.com' 2024-09-21T09:39:36.196 INFO:teuthology.run_tasks:Running task clock... 2024-09-21T09:39:36.204 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-21T09:39:36.205 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:39:36.205 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:39:36.210 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-21T09:39:36.210 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T09:39:36.234 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:39:36.249 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:39:36.263 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-21T09:39:36.274 INFO:teuthology.orchestra.run.smithi033.stderr:sudo: ntpd: command not found 2024-09-21T09:39:36.277 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-21T09:39:36.286 INFO:teuthology.orchestra.run.smithi033.stdout:506 Cannot talk to daemon 2024-09-21T09:39:36.301 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:39:36.301 INFO:teuthology.orchestra.run.smithi105.stderr:sudo: ntpd: command not found 2024-09-21T09:39:36.312 INFO:teuthology.orchestra.run.smithi105.stdout:506 Cannot talk to daemon 2024-09-21T09:39:36.315 INFO:teuthology.orchestra.run.smithi033.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:39:36.326 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-21T09:39:36.341 INFO:teuthology.orchestra.run.smithi105.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-21T09:39:36.380 INFO:teuthology.orchestra.run.smithi033.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:39:36.384 INFO:teuthology.orchestra.run.smithi033.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:39:36.384 INFO:teuthology.orchestra.run.smithi033.stdout:=============================================================================== 2024-09-21T09:39:36.384 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:36.384 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:36.384 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:36.384 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:36.397 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-21T09:39:36.400 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T09:39:36.400 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-21T09:39:36.400 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:36.400 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:36.401 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:36.401 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T09:39:36.401 INFO:teuthology.run_tasks:Running task pexec... 2024-09-21T09:39:36.411 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-21T09:39:36.411 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi033.front.sepia.ceph.com 2024-09-21T09:39:36.411 DEBUG:teuthology.orchestra.run.smithi033:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:39:36.411 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T09:39:36.412 DEBUG:teuthology.orchestra.run.smithi105:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-21T09:39:36.791 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T09:39:36.792 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout: Package Architecture Version Repository Size 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout:Removing: 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout:Removing unused dependencies: 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout:Remove 2 Packages 2024-09-21T09:39:36.793 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:39:36.794 INFO:teuthology.orchestra.run.smithi105.stdout:Freed space: 5.5 M 2024-09-21T09:39:36.794 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T09:39:36.797 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T09:39:36.797 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout: Package Architecture Version Repository Size 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout:Removing: 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout:Removing unused dependencies: 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout:Remove 2 Packages 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:39:36.802 INFO:teuthology.orchestra.run.smithi033.stdout:Freed space: 5.5 M 2024-09-21T09:39:36.803 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-09-21T09:39:36.807 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-09-21T09:39:36.807 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-09-21T09:39:36.850 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T09:39:36.969 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T09:39:36.969 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-09-21T09:39:36.969 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-09-21T09:39:36.989 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T09:39:37.031 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-09-21T09:39:37.059 INFO:teuthology.orchestra.run.smithi105.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:39:37.107 INFO:teuthology.orchestra.run.smithi033.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-21T09:39:37.119 INFO:teuthology.orchestra.run.smithi105.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:39:37.165 INFO:teuthology.orchestra.run.smithi033.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:39:37.606 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:39:37.607 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:39:37.667 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-21T09:39:37.667 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-21T09:39:37.821 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:39:37.821 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:39:37.822 INFO:teuthology.orchestra.run.smithi105.stdout:Removed: 2024-09-21T09:39:37.822 INFO:teuthology.orchestra.run.smithi105.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:39:37.822 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:39:37.822 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T09:39:37.898 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-21T09:39:37.898 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:39:37.898 INFO:teuthology.orchestra.run.smithi033.stdout:Removed: 2024-09-21T09:39:37.898 INFO:teuthology.orchestra.run.smithi033.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:39:37.898 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:39:37.898 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-09-21T09:39:38.678 INFO:teuthology.orchestra.run.smithi105.stdout:Last metadata expiration check: 0:00:50 ago on Sat 21 Sep 2024 09:38:48 AM UTC. 2024-09-21T09:39:38.763 INFO:teuthology.orchestra.run.smithi033.stdout:Last metadata expiration check: 0:00:44 ago on Sat 21 Sep 2024 09:38:54 AM UTC. 2024-09-21T09:39:38.813 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T09:39:38.814 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:39:38.814 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-09-21T09:39:38.814 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:39:38.814 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout:================================================================================ 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout:Install 7 Packages 2024-09-21T09:39:38.815 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:39:38.829 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 5.2 M 2024-09-21T09:39:38.829 INFO:teuthology.orchestra.run.smithi105.stdout:Installed size: 19 M 2024-09-21T09:39:38.829 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T09:39:38.901 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-09-21T09:39:38.902 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:39:38.902 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout:================================================================================ 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout:Install 7 Packages 2024-09-21T09:39:38.903 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:39:38.920 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 5.2 M 2024-09-21T09:39:38.920 INFO:teuthology.orchestra.run.smithi033.stdout:Installed size: 19 M 2024-09-21T09:39:38.920 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-09-21T09:39:39.334 INFO:teuthology.orchestra.run.smithi105.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 116 kB/s | 44 kB 00:00 2024-09-21T09:39:39.336 INFO:teuthology.orchestra.run.smithi033.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 389 kB/s | 44 kB 00:00 2024-09-21T09:39:39.361 INFO:teuthology.orchestra.run.smithi033.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 760 kB/s | 106 kB 00:00 2024-09-21T09:39:39.409 INFO:teuthology.orchestra.run.smithi105.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 232 kB/s | 106 kB 00:00 2024-09-21T09:39:39.411 INFO:teuthology.orchestra.run.smithi033.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 955 kB/s | 72 kB 00:00 2024-09-21T09:39:39.437 INFO:teuthology.orchestra.run.smithi033.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.1 MB/s | 84 kB 00:00 2024-09-21T09:39:39.479 INFO:teuthology.orchestra.run.smithi033.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 3.4 MB/s | 906 kB 00:00 2024-09-21T09:39:39.518 INFO:teuthology.orchestra.run.smithi105.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 777 kB/s | 84 kB 00:00 2024-09-21T09:39:39.543 INFO:teuthology.orchestra.run.smithi105.stdout:(4/7): python3-configshell-1.1.30-1.el9.noarch. 344 kB/s | 72 kB 00:00 2024-09-21T09:39:39.587 INFO:teuthology.orchestra.run.smithi033.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 4.7 MB/s | 837 kB 00:00 2024-09-21T09:39:39.685 INFO:teuthology.orchestra.run.smithi105.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 1.2 MB/s | 906 kB 00:00 2024-09-21T09:39:39.810 INFO:teuthology.orchestra.run.smithi105.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 2.8 MB/s | 837 kB 00:00 2024-09-21T09:39:39.829 INFO:teuthology.orchestra.run.smithi033.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 8.1 MB/s | 3.2 MB 00:00 2024-09-21T09:39:39.830 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:39:39.830 INFO:teuthology.orchestra.run.smithi033.stdout:Total 5.7 MB/s | 5.2 MB 00:00 2024-09-21T09:39:39.894 INFO:teuthology.orchestra.run.smithi105.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 9.0 MB/s | 3.2 MB 00:00 2024-09-21T09:39:39.894 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:39:39.894 INFO:teuthology.orchestra.run.smithi105.stdout:Total 4.9 MB/s | 5.2 MB 00:01 2024-09-21T09:39:39.946 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-09-21T09:39:39.963 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-09-21T09:39:39.963 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-09-21T09:39:40.003 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T09:39:40.018 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T09:39:40.018 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T09:39:40.292 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-09-21T09:39:40.292 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-09-21T09:39:40.310 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T09:39:40.310 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T09:39:40.783 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-09-21T09:39:40.807 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T09:39:40.875 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-21T09:39:40.896 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-21T09:39:40.908 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-21T09:39:40.936 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-21T09:39:40.942 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-21T09:39:40.976 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-21T09:39:41.160 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-21T09:39:41.187 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-21T09:39:41.192 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:39:41.224 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:39:41.661 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:39:41.667 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-21T09:39:41.687 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:39:41.693 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:39:41.808 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:39:41.820 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-21T09:39:41.873 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:39:41.885 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:39:42.797 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:39:42.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-21T09:39:42.797 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-21T09:39:42.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-21T09:39:42.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-21T09:39:42.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-21T09:39:42.798 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-21T09:39:42.837 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:39:42.837 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-21T09:39:42.837 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-21T09:39:42.837 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-21T09:39:42.837 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-21T09:39:42.837 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-21T09:39:42.837 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:39:43.179 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:39:43.187 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-09-21T09:39:43.561 DEBUG:teuthology.parallel:result is None 2024-09-21T09:39:43.595 DEBUG:teuthology.parallel:result is None 2024-09-21T09:39:43.595 INFO:teuthology.run_tasks:Running task install... 2024-09-21T09:39:43.609 DEBUG:teuthology.task.install:project ceph 2024-09-21T09:39:43.609 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:39:43.609 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:39:43.609 INFO:teuthology.task.install:Using flavor: default 2024-09-21T09:39:43.615 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-21T09:39:43.615 INFO:teuthology.task.install:extra packages: [] 2024-09-21T09:39:43.615 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:39:43.616 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:39:43.616 INFO:teuthology.packaging:ref: None 2024-09-21T09:39:43.616 INFO:teuthology.packaging:tag: None 2024-09-21T09:39:43.616 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:39:43.616 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:39:43.616 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:39:43.618 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': ['python3-pytest'], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208', 'tag': None, 'wait_for_package': False} 2024-09-21T09:39:43.618 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:39:43.618 INFO:teuthology.packaging:ref: None 2024-09-21T09:39:43.618 INFO:teuthology.packaging:tag: None 2024-09-21T09:39:43.618 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:39:43.618 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:39:43.619 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:39:43.840 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/ 2024-09-21T09:39:43.840 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1649.gfbd1a6ab 2024-09-21T09:39:43.844 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/ 2024-09-21T09:39:43.845 INFO:teuthology.task.install.rpm:Package version is 17.2.7-1649.gfbd1a6ab 2024-09-21T09:39:43.949 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:39:43.949 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:39:43.950 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:39:43.956 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/fbd1a6ab0448f011c60f6376195d5c078b0cf20b/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-21T09:39:43.956 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:39:43.956 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-21T09:39:43.978 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-21T09:39:43.978 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:39:43.978 INFO:teuthology.packaging:ref: None 2024-09-21T09:39:43.978 INFO:teuthology.packaging:tag: None 2024-09-21T09:39:43.978 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:39:43.978 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:39:43.979 DEBUG:teuthology.orchestra.run.smithi033:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:39:43.981 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, python3-pytest, python3-pytest, python3-pytest, python3-pytest on remote rpm x86_64 2024-09-21T09:39:43.981 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:39:43.982 INFO:teuthology.packaging:ref: None 2024-09-21T09:39:43.982 INFO:teuthology.packaging:tag: None 2024-09-21T09:39:43.982 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:39:43.982 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:39:43.982 DEBUG:teuthology.orchestra.run.smithi105:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-21T09:39:44.047 DEBUG:teuthology.orchestra.run.smithi105:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:39:44.078 DEBUG:teuthology.orchestra.run.smithi033:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-21T09:39:44.116 DEBUG:teuthology.orchestra.run.smithi033:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:39:44.121 DEBUG:teuthology.orchestra.run.smithi105:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-21T09:39:44.182 INFO:teuthology.orchestra.run.smithi033.stdout:check_obsoletes = 1 2024-09-21T09:39:44.184 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum clean all 2024-09-21T09:39:44.184 INFO:teuthology.orchestra.run.smithi105.stdout:check_obsoletes = 1 2024-09-21T09:39:44.186 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum clean all 2024-09-21T09:39:44.410 INFO:teuthology.orchestra.run.smithi033.stdout:45 files removed 2024-09-21T09:39:44.416 INFO:teuthology.orchestra.run.smithi105.stdout:45 files removed 2024-09-21T09:39:44.438 DEBUG:teuthology.orchestra.run.smithi033:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-21T09:39:44.443 DEBUG:teuthology.orchestra.run.smithi105:> sudo yum -y install ceph-radosgw ceph-test ceph ceph-base cephadm ceph-immutable-object-cache ceph-mgr ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-rook ceph-mgr-cephadm ceph-fuse librados-devel libcephfs2 libcephfs-devel librados2 librbd1 python3-rados python3-rgw python3-cephfs python3-rbd rbd-fuse rbd-mirror rbd-nbd python3-pytest python3-pytest python3-pytest python3-pytest 2024-09-21T09:39:45.002 INFO:teuthology.orchestra.run.smithi105.stdout:ceph packages for x86_64 285 kB/s | 77 kB 00:00 2024-09-21T09:39:45.010 INFO:teuthology.orchestra.run.smithi033.stdout:ceph packages for x86_64 283 kB/s | 77 kB 00:00 2024-09-21T09:39:45.285 INFO:teuthology.orchestra.run.smithi105.stdout:ceph noarch packages 46 kB/s | 11 kB 00:00 2024-09-21T09:39:45.294 INFO:teuthology.orchestra.run.smithi033.stdout:ceph noarch packages 45 kB/s | 11 kB 00:00 2024-09-21T09:39:45.561 INFO:teuthology.orchestra.run.smithi105.stdout:ceph source packages 7.4 kB/s | 1.8 kB 00:00 2024-09-21T09:39:45.570 INFO:teuthology.orchestra.run.smithi033.stdout:ceph source packages 7.5 kB/s | 1.8 kB 00:00 2024-09-21T09:39:46.336 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - BaseOS 11 MB/s | 8.3 MB 00:00 2024-09-21T09:39:47.478 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - BaseOS 4.4 MB/s | 8.3 MB 00:01 2024-09-21T09:39:48.104 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - AppStream 29 MB/s | 20 MB 00:00 2024-09-21T09:39:51.046 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - AppStream 8.4 MB/s | 20 MB 00:02 2024-09-21T09:39:53.639 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - CRB 3.7 MB/s | 6.5 MB 00:01 2024-09-21T09:39:55.264 INFO:teuthology.orchestra.run.smithi105.stdout:CentOS Stream 9 - Extras packages 48 kB/s | 19 kB 00:00 2024-09-21T09:39:56.160 INFO:teuthology.orchestra.run.smithi105.stdout:Extra Packages for Enterprise Linux 29 MB/s | 23 MB 00:00 2024-09-21T09:39:57.538 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - CRB 2.5 MB/s | 6.5 MB 00:02 2024-09-21T09:39:58.997 INFO:teuthology.orchestra.run.smithi033.stdout:CentOS Stream 9 - Extras packages 107 kB/s | 19 kB 00:00 2024-09-21T09:39:59.918 INFO:teuthology.orchestra.run.smithi033.stdout:Extra Packages for Enterprise Linux 29 MB/s | 23 MB 00:00 2024-09-21T09:40:01.524 INFO:teuthology.orchestra.run.smithi105.stdout:lab-extras 28 kB/s | 1.7 kB 00:00 2024-09-21T09:40:03.852 INFO:teuthology.orchestra.run.smithi105.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-21T09:40:03.853 INFO:teuthology.orchestra.run.smithi105.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-21T09:40:03.916 INFO:teuthology.orchestra.run.smithi105.stdout:Dependencies resolved. 2024-09-21T09:40:03.922 INFO:teuthology.orchestra.run.smithi105.stdout:======================================================================================= 2024-09-21T09:40:03.922 INFO:teuthology.orchestra.run.smithi105.stdout: Package Arch Version Repository Size 2024-09-21T09:40:03.922 INFO:teuthology.orchestra.run.smithi105.stdout:======================================================================================= 2024-09-21T09:40:03.922 INFO:teuthology.orchestra.run.smithi105.stdout:Installing: 2024-09-21T09:40:03.922 INFO:teuthology.orchestra.run.smithi105.stdout: ceph x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 6.5 k 2024-09-21T09:40:03.922 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-base x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 5.0 M 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 813 k 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 137 k 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 1.4 M 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 124 k 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 3.4 M 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 7.4 M 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-rook noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 47 k 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-radosgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 14 M 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-test x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 48 M 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 83 k 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 636 k 2024-09-21T09:40:03.923 INFO:teuthology.orchestra.run.smithi105.stdout: librados-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 126 k 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cephfs x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 156 k 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rados x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 320 k 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 295 k 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 99 k 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 82 k 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-mirror x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-nbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 165 k 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout:Upgrading: 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: librados2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.1 M 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: librbd1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout:Installing dependencies: 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 22 M 2024-09-21T09:40:03.924 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 24 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mds x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.1 M 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 240 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mon x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.9 M 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-osd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 15 M 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 15 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-selinux x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: libcephsqlite x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 163 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:40:03.925 INFO:teuthology.orchestra.run.smithi105.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: libradosstriper1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 457 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: librgw2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 4.9 M 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:40:03.926 INFO:teuthology.orchestra.run.smithi105.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-argparse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 45 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 120 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-21T09:40:03.927 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:40:03.928 INFO:teuthology.orchestra.run.smithi105.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:40:03.929 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:40:03.930 INFO:teuthology.orchestra.run.smithi105.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout:Installing weak dependencies: 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction Summary 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout:======================================================================================= 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout:Install 112 Packages 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout:Upgrade 2 Packages 2024-09-21T09:40:03.931 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:03.932 INFO:teuthology.orchestra.run.smithi105.stdout:Total download size: 190 M 2024-09-21T09:40:03.932 INFO:teuthology.orchestra.run.smithi105.stdout:Downloading Packages: 2024-09-21T09:40:04.334 INFO:teuthology.orchestra.run.smithi105.stdout:(1/114): ceph-17.2.7-1649.gfbd1a6ab.el9.x86_64. 50 kB/s | 6.5 kB 00:00 2024-09-21T09:40:04.535 INFO:teuthology.orchestra.run.smithi105.stdout:(2/114): ceph-fuse-17.2.7-1649.gfbd1a6ab.el9.x8 4.0 MB/s | 813 kB 00:00 2024-09-21T09:40:04.585 INFO:teuthology.orchestra.run.smithi105.stdout:(3/114): ceph-immutable-object-cache-17.2.7-164 2.7 MB/s | 137 kB 00:00 2024-09-21T09:40:04.793 INFO:teuthology.orchestra.run.smithi105.stdout:(4/114): ceph-mds-17.2.7-1649.gfbd1a6ab.el9.x86 10 MB/s | 2.1 MB 00:00 2024-09-21T09:40:04.893 INFO:teuthology.orchestra.run.smithi105.stdout:(5/114): ceph-base-17.2.7-1649.gfbd1a6ab.el9.x8 7.2 MB/s | 5.0 MB 00:00 2024-09-21T09:40:04.977 INFO:teuthology.orchestra.run.smithi105.stdout:(6/114): ceph-mgr-17.2.7-1649.gfbd1a6ab.el9.x86 7.7 MB/s | 1.4 MB 00:00 2024-09-21T09:40:05.203 INFO:teuthology.orchestra.run.smithi105.stdout:(7/114): ceph-mon-17.2.7-1649.gfbd1a6ab.el9.x86 13 MB/s | 3.9 MB 00:00 2024-09-21T09:40:05.440 INFO:teuthology.orchestra.run.smithi033.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-21T09:40:05.828 INFO:teuthology.orchestra.run.smithi105.stdout:(8/114): ceph-common-17.2.7-1649.gfbd1a6ab.el9. 14 MB/s | 22 MB 00:01 2024-09-21T09:40:05.878 INFO:teuthology.orchestra.run.smithi105.stdout:(9/114): ceph-selinux-17.2.7-1649.gfbd1a6ab.el9 507 kB/s | 25 kB 00:00 2024-09-21T09:40:06.062 INFO:teuthology.orchestra.run.smithi105.stdout:(10/114): ceph-osd-17.2.7-1649.gfbd1a6ab.el9.x8 14 MB/s | 15 MB 00:01 2024-09-21T09:40:06.229 INFO:teuthology.orchestra.run.smithi105.stdout:(11/114): ceph-radosgw-17.2.7-1649.gfbd1a6ab.el 14 MB/s | 14 MB 00:01 2024-09-21T09:40:06.245 INFO:teuthology.orchestra.run.smithi105.stdout:(12/114): libcephfs-devel-17.2.7-1649.gfbd1a6ab 136 kB/s | 25 kB 00:00 2024-09-21T09:40:06.296 INFO:teuthology.orchestra.run.smithi105.stdout:(13/114): libcephfs2-17.2.7-1649.gfbd1a6ab.el9. 9.3 MB/s | 636 kB 00:00 2024-09-21T09:40:06.321 INFO:teuthology.orchestra.run.smithi105.stdout:(14/114): libcephsqlite-17.2.7-1649.gfbd1a6ab.e 2.1 MB/s | 163 kB 00:00 2024-09-21T09:40:06.355 INFO:teuthology.orchestra.run.smithi105.stdout:(15/114): librados-devel-17.2.7-1649.gfbd1a6ab. 2.1 MB/s | 126 kB 00:00 2024-09-21T09:40:06.388 INFO:teuthology.orchestra.run.smithi105.stdout:(16/114): libradosstriper1-17.2.7-1649.gfbd1a6a 6.7 MB/s | 457 kB 00:00 2024-09-21T09:40:06.455 INFO:teuthology.orchestra.run.smithi105.stdout:(17/114): python3-ceph-argparse-17.2.7-1649.gfb 679 kB/s | 45 kB 00:00 2024-09-21T09:40:06.531 INFO:teuthology.orchestra.run.smithi105.stdout:(18/114): python3-ceph-common-17.2.7-1649.gfbd1 1.6 MB/s | 120 kB 00:00 2024-09-21T09:40:06.606 INFO:teuthology.orchestra.run.smithi105.stdout:(19/114): python3-cephfs-17.2.7-1649.gfbd1a6ab. 2.0 MB/s | 156 kB 00:00 2024-09-21T09:40:06.690 INFO:teuthology.orchestra.run.smithi105.stdout:(20/114): librgw2-17.2.7-1649.gfbd1a6ab.el9.x86 15 MB/s | 4.9 MB 00:00 2024-09-21T09:40:06.723 INFO:teuthology.orchestra.run.smithi105.stdout:(21/114): python3-rados-17.2.7-1649.gfbd1a6ab.e 2.7 MB/s | 320 kB 00:00 2024-09-21T09:40:06.748 INFO:teuthology.orchestra.run.smithi105.stdout:(22/114): python3-rbd-17.2.7-1649.gfbd1a6ab.el9 4.9 MB/s | 295 kB 00:00 2024-09-21T09:40:06.774 INFO:teuthology.orchestra.run.smithi105.stdout:(23/114): python3-rgw-17.2.7-1649.gfbd1a6ab.el9 1.9 MB/s | 99 kB 00:00 2024-09-21T09:40:06.799 INFO:teuthology.orchestra.run.smithi105.stdout:(24/114): rbd-fuse-17.2.7-1649.gfbd1a6ab.el9.x8 1.6 MB/s | 82 kB 00:00 2024-09-21T09:40:06.858 INFO:teuthology.orchestra.run.smithi105.stdout:(25/114): rbd-nbd-17.2.7-1649.gfbd1a6ab.el9.x86 2.8 MB/s | 165 kB 00:00 2024-09-21T09:40:06.933 INFO:teuthology.orchestra.run.smithi105.stdout:(26/114): rbd-mirror-17.2.7-1649.gfbd1a6ab.el9. 18 MB/s | 2.9 MB 00:00 2024-09-21T09:40:06.960 INFO:teuthology.orchestra.run.smithi105.stdout:(27/114): ceph-grafana-dashboards-17.2.7-1649.g 240 kB/s | 24 kB 00:00 2024-09-21T09:40:06.985 INFO:teuthology.orchestra.run.smithi105.stdout:(28/114): ceph-mgr-cephadm-17.2.7-1649.gfbd1a6a 2.4 MB/s | 124 kB 00:00 2024-09-21T09:40:07.419 INFO:teuthology.orchestra.run.smithi105.stdout:(29/114): ceph-mgr-dashboard-17.2.7-1649.gfbd1a 7.3 MB/s | 3.4 MB 00:00 2024-09-21T09:40:07.511 INFO:teuthology.orchestra.run.smithi105.stdout:(30/114): ceph-mgr-diskprediction-local-17.2.7- 14 MB/s | 7.4 MB 00:00 2024-09-21T09:40:07.561 INFO:teuthology.orchestra.run.smithi105.stdout:(31/114): ceph-mgr-rook-17.2.7-1649.gfbd1a6ab.e 953 kB/s | 47 kB 00:00 2024-09-21T09:40:07.586 INFO:teuthology.orchestra.run.smithi105.stdout:(32/114): ceph-mgr-modules-core-17.2.7-1649.gfb 1.4 MB/s | 240 kB 00:00 2024-09-21T09:40:07.612 INFO:teuthology.orchestra.run.smithi105.stdout:(33/114): ceph-prometheus-alerts-17.2.7-1649.gf 289 kB/s | 15 kB 00:00 2024-09-21T09:40:07.637 INFO:teuthology.orchestra.run.smithi105.stdout:(34/114): cephadm-17.2.7-1649.gfbd1a6ab.el9.noa 1.6 MB/s | 83 kB 00:00 2024-09-21T09:40:07.777 INFO:teuthology.orchestra.run.smithi033.stdout:Package librados2-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-21T09:40:07.778 INFO:teuthology.orchestra.run.smithi033.stdout:Package librbd1-2:16.2.4-5.el9.x86_64 is already installed. 2024-09-21T09:40:07.848 INFO:teuthology.orchestra.run.smithi033.stdout:Dependencies resolved. 2024-09-21T09:40:07.855 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-09-21T09:40:07.855 INFO:teuthology.orchestra.run.smithi033.stdout: Package Arch Version Repository Size 2024-09-21T09:40:07.855 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout:Installing: 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 6.5 k 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-base x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 5.0 M 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 813 k 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 137 k 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 1.4 M 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 124 k 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-dashboard noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 3.4 M 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 7.4 M 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-rook noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 47 k 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-radosgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 14 M 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-test x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 48 M 2024-09-21T09:40:07.856 INFO:teuthology.orchestra.run.smithi033.stdout: cephadm noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 83 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 636 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: librados-devel x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 126 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cephfs x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 156 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pytest noarch 6.2.2-7.el9 appstream 519 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rados x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 320 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 295 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rgw x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 99 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-fuse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 82 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-mirror x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-nbd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 165 k 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout:Upgrading: 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: librados2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.1 M 2024-09-21T09:40:07.857 INFO:teuthology.orchestra.run.smithi033.stdout: librbd1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.9 M 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout:Installing dependencies: 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 22 M 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-grafana-dashboards noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 24 k 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mds x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 2.1 M 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-modules-core noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 240 k 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mon x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 3.9 M 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-osd x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 15 M 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-prometheus-alerts noarch 2:17.2.7-1649.gfbd1a6ab.el9 ceph-noarch 15 k 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-selinux x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 25 k 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-21T09:40:07.858 INFO:teuthology.orchestra.run.smithi033.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: libcephsqlite x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 163 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: libradosstriper1 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 457 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: librgw2 x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 4.9 M 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-21T09:40:07.859 INFO:teuthology.orchestra.run.smithi033.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-attrs noarch 20.3.0-7.el9 appstream 87 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-argparse x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 45 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-common x86_64 2:17.2.7-1649.gfbd1a6ab.el9 ceph 120 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-21T09:40:07.860 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-iniconfig noarch 1.1.1-7.el9 appstream 17 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-21T09:40:07.861 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pluggy noarch 0.13.1-7.el9 appstream 41 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-21T09:40:07.862 INFO:teuthology.orchestra.run.smithi033.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-py noarch 1.10.0-6.el9 appstream 477 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-21T09:40:07.863 INFO:teuthology.orchestra.run.smithi033.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout:Installing weak dependencies: 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction Summary 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout:======================================================================================= 2024-09-21T09:40:07.864 INFO:teuthology.orchestra.run.smithi033.stdout:Install 112 Packages 2024-09-21T09:40:07.865 INFO:teuthology.orchestra.run.smithi033.stdout:Upgrade 2 Packages 2024-09-21T09:40:07.865 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:07.865 INFO:teuthology.orchestra.run.smithi033.stdout:Total download size: 190 M 2024-09-21T09:40:07.865 INFO:teuthology.orchestra.run.smithi033.stdout:Downloading Packages: 2024-09-21T09:40:08.063 INFO:teuthology.orchestra.run.smithi105.stdout:(35/114): ceph-test-17.2.7-1649.gfbd1a6ab.el9.x 22 MB/s | 48 MB 00:02 2024-09-21T09:40:08.208 INFO:teuthology.orchestra.run.smithi033.stdout:(1/114): ceph-17.2.7-1649.gfbd1a6ab.el9.x86_64. 52 kB/s | 6.5 kB 00:00 2024-09-21T09:40:08.271 INFO:teuthology.orchestra.run.smithi105.stdout:(36/114): fuse-2.9.9-16.el9.x86_64.rpm 121 kB/s | 80 kB 00:00 2024-09-21T09:40:08.400 INFO:teuthology.orchestra.run.smithi033.stdout:(2/114): ceph-fuse-17.2.7-1649.gfbd1a6ab.el9.x8 4.2 MB/s | 813 kB 00:00 2024-09-21T09:40:08.450 INFO:teuthology.orchestra.run.smithi033.stdout:(3/114): ceph-immutable-object-cache-17.2.7-164 2.7 MB/s | 137 kB 00:00 2024-09-21T09:40:08.455 INFO:teuthology.orchestra.run.smithi105.stdout:(37/114): libquadmath-11.5.0-2.el9.x86_64.rpm 488 kB/s | 191 kB 00:00 2024-09-21T09:40:08.522 INFO:teuthology.orchestra.run.smithi105.stdout:(38/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.0 MB/s | 253 kB 00:00 2024-09-21T09:40:08.597 INFO:teuthology.orchestra.run.smithi105.stdout:(39/114): python3-ply-3.11-14.el9.noarch.rpm 1.4 MB/s | 106 kB 00:00 2024-09-21T09:40:08.667 INFO:teuthology.orchestra.run.smithi033.stdout:(4/114): ceph-mds-17.2.7-1649.gfbd1a6ab.el9.x86 9.6 MB/s | 2.1 MB 00:00 2024-09-21T09:40:08.714 INFO:teuthology.orchestra.run.smithi105.stdout:(40/114): libgfortran-11.5.0-2.el9.x86_64.rpm 744 kB/s | 801 kB 00:01 2024-09-21T09:40:08.739 INFO:teuthology.orchestra.run.smithi105.stdout:(41/114): python3-pycparser-2.20-6.el9.noarch.r 952 kB/s | 135 kB 00:00 2024-09-21T09:40:08.742 INFO:teuthology.orchestra.run.smithi033.stdout:(5/114): ceph-base-17.2.7-1649.gfbd1a6ab.el9.x8 7.5 MB/s | 5.0 MB 00:00 2024-09-21T09:40:08.834 INFO:teuthology.orchestra.run.smithi033.stdout:(6/114): ceph-mgr-17.2.7-1649.gfbd1a6ab.el9.x86 8.4 MB/s | 1.4 MB 00:00 2024-09-21T09:40:08.840 INFO:teuthology.orchestra.run.smithi105.stdout:(42/114): python3-requests-2.25.1-8.el9.noarch. 997 kB/s | 125 kB 00:00 2024-09-21T09:40:08.873 INFO:teuthology.orchestra.run.smithi105.stdout:(43/114): python3-urllib3-1.26.5-6.el9.noarch.r 1.6 MB/s | 215 kB 00:00 2024-09-21T09:40:09.043 INFO:teuthology.orchestra.run.smithi033.stdout:(7/114): ceph-mon-17.2.7-1649.gfbd1a6ab.el9.x86 13 MB/s | 3.9 MB 00:00 2024-09-21T09:40:09.099 INFO:teuthology.orchestra.run.smithi105.stdout:(44/114): python3-cryptography-36.0.1-4.el9.x86 1.9 MB/s | 1.2 MB 00:00 2024-09-21T09:40:09.391 INFO:teuthology.orchestra.run.smithi105.stdout:(45/114): flexiblas-3.0.4-8.el9.x86_64.rpm 59 kB/s | 33 kB 00:00 2024-09-21T09:40:09.583 INFO:teuthology.orchestra.run.smithi105.stdout:(46/114): flexiblas-openblas-openmp-3.0.4-8.el9 38 kB/s | 18 kB 00:00 2024-09-21T09:40:09.717 INFO:teuthology.orchestra.run.smithi033.stdout:(8/114): ceph-common-17.2.7-1649.gfbd1a6ab.el9. 14 MB/s | 22 MB 00:01 2024-09-21T09:40:09.769 INFO:teuthology.orchestra.run.smithi033.stdout:(9/114): ceph-selinux-17.2.7-1649.gfbd1a6ab.el9 496 kB/s | 25 kB 00:00 2024-09-21T09:40:09.925 INFO:teuthology.orchestra.run.smithi105.stdout:(47/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 2.8 MB/s | 3.0 MB 00:01 2024-09-21T09:40:09.944 INFO:teuthology.orchestra.run.smithi033.stdout:(10/114): ceph-osd-17.2.7-1649.gfbd1a6ab.el9.x8 14 MB/s | 15 MB 00:01 2024-09-21T09:40:10.111 INFO:teuthology.orchestra.run.smithi033.stdout:(11/114): ceph-radosgw-17.2.7-1649.gfbd1a6ab.el 14 MB/s | 14 MB 00:01 2024-09-21T09:40:10.125 INFO:teuthology.orchestra.run.smithi105.stdout:(48/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 218 kB/s | 160 kB 00:00 2024-09-21T09:40:10.136 INFO:teuthology.orchestra.run.smithi033.stdout:(12/114): libcephfs-devel-17.2.7-1649.gfbd1a6ab 130 kB/s | 25 kB 00:00 2024-09-21T09:40:10.170 INFO:teuthology.orchestra.run.smithi033.stdout:(13/114): libcephfs2-17.2.7-1649.gfbd1a6ab.el9. 11 MB/s | 636 kB 00:00 2024-09-21T09:40:10.195 INFO:teuthology.orchestra.run.smithi033.stdout:(14/114): libcephsqlite-17.2.7-1649.gfbd1a6ab.e 2.7 MB/s | 163 kB 00:00 2024-09-21T09:40:10.208 INFO:teuthology.orchestra.run.smithi105.stdout:(49/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 72 kB/s | 45 kB 00:00 2024-09-21T09:40:10.220 INFO:teuthology.orchestra.run.smithi033.stdout:(15/114): librados-devel-17.2.7-1649.gfbd1a6ab. 2.5 MB/s | 126 kB 00:00 2024-09-21T09:40:10.254 INFO:teuthology.orchestra.run.smithi033.stdout:(16/114): libradosstriper1-17.2.7-1649.gfbd1a6a 7.6 MB/s | 457 kB 00:00 2024-09-21T09:40:10.321 INFO:teuthology.orchestra.run.smithi033.stdout:(17/114): python3-ceph-argparse-17.2.7-1649.gfb 681 kB/s | 45 kB 00:00 2024-09-21T09:40:10.396 INFO:teuthology.orchestra.run.smithi033.stdout:(18/114): python3-ceph-common-17.2.7-1649.gfbd1 1.6 MB/s | 120 kB 00:00 2024-09-21T09:40:10.471 INFO:teuthology.orchestra.run.smithi033.stdout:(19/114): librgw2-17.2.7-1649.gfbd1a6ab.el9.x86 20 MB/s | 4.9 MB 00:00 2024-09-21T09:40:10.497 INFO:teuthology.orchestra.run.smithi033.stdout:(20/114): python3-cephfs-17.2.7-1649.gfbd1a6ab. 1.5 MB/s | 156 kB 00:00 2024-09-21T09:40:10.595 INFO:teuthology.orchestra.run.smithi033.stdout:(21/114): python3-rados-17.2.7-1649.gfbd1a6ab.e 2.5 MB/s | 320 kB 00:00 2024-09-21T09:40:10.655 INFO:teuthology.orchestra.run.smithi033.stdout:(22/114): python3-rbd-17.2.7-1649.gfbd1a6ab.el9 1.8 MB/s | 295 kB 00:00 2024-09-21T09:40:10.681 INFO:teuthology.orchestra.run.smithi033.stdout:(23/114): python3-rgw-17.2.7-1649.gfbd1a6ab.el9 1.1 MB/s | 99 kB 00:00 2024-09-21T09:40:10.706 INFO:teuthology.orchestra.run.smithi033.stdout:(24/114): rbd-fuse-17.2.7-1649.gfbd1a6ab.el9.x8 1.6 MB/s | 82 kB 00:00 2024-09-21T09:40:10.765 INFO:teuthology.orchestra.run.smithi033.stdout:(25/114): rbd-nbd-17.2.7-1649.gfbd1a6ab.el9.x86 2.8 MB/s | 165 kB 00:00 2024-09-21T09:40:10.775 INFO:teuthology.orchestra.run.smithi105.stdout:(50/114): openblas-0.3.26-2.el9.x86_64.rpm 68 kB/s | 38 kB 00:00 2024-09-21T09:40:10.800 INFO:teuthology.orchestra.run.smithi105.stdout:(51/114): librdkafka-1.6.1-102.el9.x86_64.rpm 757 kB/s | 662 kB 00:00 2024-09-21T09:40:10.832 INFO:teuthology.orchestra.run.smithi033.stdout:(26/114): rbd-mirror-17.2.7-1649.gfbd1a6ab.el9. 19 MB/s | 2.9 MB 00:00 2024-09-21T09:40:10.857 INFO:teuthology.orchestra.run.smithi033.stdout:(27/114): ceph-grafana-dashboards-17.2.7-1649.g 266 kB/s | 24 kB 00:00 2024-09-21T09:40:10.882 INFO:teuthology.orchestra.run.smithi033.stdout:(28/114): ceph-mgr-cephadm-17.2.7-1649.gfbd1a6a 2.4 MB/s | 124 kB 00:00 2024-09-21T09:40:10.926 INFO:teuthology.orchestra.run.smithi105.stdout:(52/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 365 kB/s | 292 kB 00:00 2024-09-21T09:40:11.083 INFO:teuthology.orchestra.run.smithi033.stdout:(29/114): ceph-mgr-dashboard-17.2.7-1649.gfbd1a 15 MB/s | 3.4 MB 00:00 2024-09-21T09:40:11.184 INFO:teuthology.orchestra.run.smithi033.stdout:(30/114): ceph-mgr-modules-core-17.2.7-1649.gfb 2.3 MB/s | 240 kB 00:00 2024-09-21T09:40:11.284 INFO:teuthology.orchestra.run.smithi033.stdout:(31/114): ceph-mgr-diskprediction-local-17.2.7- 18 MB/s | 7.4 MB 00:00 2024-09-21T09:40:11.309 INFO:teuthology.orchestra.run.smithi033.stdout:(32/114): ceph-mgr-rook-17.2.7-1649.gfbd1a6ab.e 380 kB/s | 47 kB 00:00 2024-09-21T09:40:11.334 INFO:teuthology.orchestra.run.smithi033.stdout:(33/114): ceph-prometheus-alerts-17.2.7-1649.gf 291 kB/s | 15 kB 00:00 2024-09-21T09:40:11.360 INFO:teuthology.orchestra.run.smithi033.stdout:(34/114): cephadm-17.2.7-1649.gfbd1a6ab.el9.noa 1.6 MB/s | 83 kB 00:00 2024-09-21T09:40:11.443 INFO:teuthology.orchestra.run.smithi105.stdout:(53/114): python3-attrs-20.3.0-7.el9.noarch.rpm 136 kB/s | 87 kB 00:00 2024-09-21T09:40:11.510 INFO:teuthology.orchestra.run.smithi033.stdout:(35/114): fuse-2.9.9-16.el9.x86_64.rpm 456 kB/s | 80 kB 00:00 2024-09-21T09:40:11.594 INFO:teuthology.orchestra.run.smithi033.stdout:(36/114): libquadmath-11.5.0-2.el9.x86_64.rpm 2.2 MB/s | 191 kB 00:00 2024-09-21T09:40:11.627 INFO:teuthology.orchestra.run.smithi033.stdout:(37/114): libgfortran-11.5.0-2.el9.x86_64.rpm 2.9 MB/s | 801 kB 00:00 2024-09-21T09:40:11.653 INFO:teuthology.orchestra.run.smithi033.stdout:(38/114): python3-cffi-1.14.5-5.el9.x86_64.rpm 4.2 MB/s | 253 kB 00:00 2024-09-21T09:40:11.910 INFO:teuthology.orchestra.run.smithi105.stdout:(54/114): openblas-openmp-0.3.26-2.el9.x86_64.r 4.4 MB/s | 4.9 MB 00:01 2024-09-21T09:40:12.085 INFO:teuthology.orchestra.run.smithi105.stdout:(55/114): python3-babel-2.9.1-2.el9.noarch.rpm 5.1 MB/s | 6.0 MB 00:01 2024-09-21T09:40:12.120 INFO:teuthology.orchestra.run.smithi033.stdout:(39/114): ceph-test-17.2.7-1649.gfbd1a6ab.el9.x 20 MB/s | 48 MB 00:02 2024-09-21T09:40:12.145 INFO:teuthology.orchestra.run.smithi033.stdout:(40/114): python3-ply-3.11-14.el9.noarch.rpm 216 kB/s | 106 kB 00:00 2024-09-21T09:40:12.178 INFO:teuthology.orchestra.run.smithi033.stdout:(41/114): python3-cryptography-36.0.1-4.el9.x86 2.2 MB/s | 1.2 MB 00:00 2024-09-21T09:40:12.203 INFO:teuthology.orchestra.run.smithi033.stdout:(42/114): python3-requests-2.25.1-8.el9.noarch. 2.1 MB/s | 125 kB 00:00 2024-09-21T09:40:12.219 INFO:teuthology.orchestra.run.smithi105.stdout:(56/114): python3-devel-3.9.19-8.el9.x86_64.rpm 316 kB/s | 245 kB 00:00 2024-09-21T09:40:12.237 INFO:teuthology.orchestra.run.smithi033.stdout:(43/114): python3-urllib3-1.26.5-6.el9.noarch.r 3.6 MB/s | 215 kB 00:00 2024-09-21T09:40:12.346 INFO:teuthology.orchestra.run.smithi033.stdout:(44/114): python3-pycparser-2.20-6.el9.noarch.r 599 kB/s | 135 kB 00:00 2024-09-21T09:40:12.427 INFO:teuthology.orchestra.run.smithi105.stdout:(57/114): python3-iniconfig-1.1.1-7.el9.noarch. 34 kB/s | 17 kB 00:00 2024-09-21T09:40:12.488 INFO:teuthology.orchestra.run.smithi033.stdout:(45/114): flexiblas-3.0.4-8.el9.x86_64.rpm 115 kB/s | 33 kB 00:00 2024-09-21T09:40:12.630 INFO:teuthology.orchestra.run.smithi033.stdout:(46/114): flexiblas-openblas-openmp-3.0.4-8.el9 64 kB/s | 18 kB 00:00 2024-09-21T09:40:12.753 INFO:teuthology.orchestra.run.smithi105.stdout:(58/114): python3-jsonpatch-1.21-16.el9.noarch. 48 kB/s | 26 kB 00:00 2024-09-21T09:40:12.838 INFO:teuthology.orchestra.run.smithi033.stdout:(47/114): librabbitmq-0.11.0-7.el9.x86_64.rpm 217 kB/s | 45 kB 00:00 2024-09-21T09:40:12.861 INFO:teuthology.orchestra.run.smithi105.stdout:(59/114): python3-jinja2-2.11.3-6.el9.noarch.rp 320 kB/s | 248 kB 00:00 2024-09-21T09:40:12.864 INFO:teuthology.orchestra.run.smithi033.stdout:(48/114): libpmemobj-1.12.1-1.el9.x86_64.rpm 426 kB/s | 160 kB 00:00 2024-09-21T09:40:12.911 INFO:teuthology.orchestra.run.smithi105.stdout:(60/114): python3-jsonpointer-2.0-4.el9.noarch. 40 kB/s | 19 kB 00:00 2024-09-21T09:40:13.064 INFO:teuthology.orchestra.run.smithi033.stdout:(49/114): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.4 MB/s | 292 kB 00:00 2024-09-21T09:40:13.173 INFO:teuthology.orchestra.run.smithi033.stdout:(50/114): openblas-0.3.26-2.el9.x86_64.rpm 355 kB/s | 38 kB 00:00 2024-09-21T09:40:13.389 INFO:teuthology.orchestra.run.smithi033.stdout:(51/114): librdkafka-1.6.1-102.el9.x86_64.rpm 1.2 MB/s | 662 kB 00:00 2024-09-21T09:40:13.412 INFO:teuthology.orchestra.run.smithi105.stdout:(61/114): python3-markupsafe-1.1.1-12.el9.x86_6 63 kB/s | 35 kB 00:00 2024-09-21T09:40:13.495 INFO:teuthology.orchestra.run.smithi105.stdout:(62/114): python3-mako-1.1.4-6.el9.noarch.rpm 232 kB/s | 172 kB 00:00 2024-09-21T09:40:13.498 INFO:teuthology.orchestra.run.smithi033.stdout:(52/114): python3-attrs-20.3.0-7.el9.noarch.rpm 807 kB/s | 87 kB 00:00 2024-09-21T09:40:13.582 INFO:teuthology.orchestra.run.smithi033.stdout:(53/114): flexiblas-netlib-3.0.4-8.el9.x86_64.r 2.2 MB/s | 3.0 MB 00:01 2024-09-21T09:40:13.715 INFO:teuthology.orchestra.run.smithi033.stdout:(54/114): openblas-openmp-0.3.26-2.el9.x86_64.r 9.1 MB/s | 4.9 MB 00:00 2024-09-21T09:40:13.740 INFO:teuthology.orchestra.run.smithi033.stdout:(55/114): python3-devel-3.9.19-8.el9.x86_64.rpm 1.5 MB/s | 245 kB 00:00 2024-09-21T09:40:13.841 INFO:teuthology.orchestra.run.smithi033.stdout:(56/114): python3-iniconfig-1.1.1-7.el9.noarch. 139 kB/s | 17 kB 00:00 2024-09-21T09:40:13.866 INFO:teuthology.orchestra.run.smithi033.stdout:(57/114): python3-jinja2-2.11.3-6.el9.noarch.rp 1.9 MB/s | 248 kB 00:00 2024-09-21T09:40:13.983 INFO:teuthology.orchestra.run.smithi033.stdout:(58/114): python3-babel-2.9.1-2.el9.noarch.rpm 12 MB/s | 6.0 MB 00:00 2024-09-21T09:40:14.008 INFO:teuthology.orchestra.run.smithi033.stdout:(59/114): python3-jsonpatch-1.21-16.el9.noarch. 153 kB/s | 26 kB 00:00 2024-09-21T09:40:14.029 INFO:teuthology.orchestra.run.smithi105.stdout:(63/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 4.6 MB/s | 5.1 MB 00:01 2024-09-21T09:40:14.033 INFO:teuthology.orchestra.run.smithi033.stdout:(60/114): python3-jsonpointer-2.0-4.el9.noarch. 115 kB/s | 19 kB 00:00 2024-09-21T09:40:14.101 INFO:teuthology.orchestra.run.smithi033.stdout:(61/114): python3-mako-1.1.4-6.el9.noarch.rpm 1.4 MB/s | 172 kB 00:00 2024-09-21T09:40:14.142 INFO:teuthology.orchestra.run.smithi033.stdout:(62/114): python3-markupsafe-1.1.1-12.el9.x86_6 320 kB/s | 35 kB 00:00 2024-09-21T09:40:14.218 INFO:teuthology.orchestra.run.smithi033.stdout:(63/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 2.1 MB/s | 250 kB 00:00 2024-09-21T09:40:14.237 INFO:teuthology.orchestra.run.smithi105.stdout:(64/114): python3-numpy-f2py-1.20.1-5.el9.x86_6 304 kB/s | 250 kB 00:00 2024-09-21T09:40:14.260 INFO:teuthology.orchestra.run.smithi033.stdout:(64/114): python3-oauthlib-3.1.1-5.el9.noarch.r 1.9 MB/s | 222 kB 00:00 2024-09-21T09:40:14.263 INFO:teuthology.orchestra.run.smithi105.stdout:(65/114): python3-oauthlib-3.1.1-5.el9.noarch.r 290 kB/s | 222 kB 00:00 2024-09-21T09:40:14.327 INFO:teuthology.orchestra.run.smithi033.stdout:(65/114): python3-packaging-20.9-5.el9.noarch.r 711 kB/s | 77 kB 00:00 2024-09-21T09:40:14.385 INFO:teuthology.orchestra.run.smithi033.stdout:(66/114): python3-pluggy-0.13.1-7.el9.noarch.rp 331 kB/s | 41 kB 00:00 2024-09-21T09:40:14.452 INFO:teuthology.orchestra.run.smithi033.stdout:(67/114): python3-numpy-1.20.1-5.el9.x86_64.rpm 12 MB/s | 5.1 MB 00:00 2024-09-21T09:40:14.477 INFO:teuthology.orchestra.run.smithi033.stdout:(68/114): python3-prettytable-0.7.2-27.el9.noar 281 kB/s | 42 kB 00:00 2024-09-21T09:40:14.511 INFO:teuthology.orchestra.run.smithi033.stdout:(69/114): python3-py-1.10.0-6.el9.noarch.rpm 3.7 MB/s | 477 kB 00:00 2024-09-21T09:40:14.570 INFO:teuthology.orchestra.run.smithi033.stdout:(70/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.3 MB/s | 159 kB 00:00 2024-09-21T09:40:14.595 INFO:teuthology.orchestra.run.smithi033.stdout:(71/114): python3-pyasn1-modules-0.4.8-6.el9.no 2.3 MB/s | 279 kB 00:00 2024-09-21T09:40:14.645 INFO:teuthology.orchestra.run.smithi033.stdout:(72/114): python3-pytest-6.2.2-7.el9.noarch.rpm 3.8 MB/s | 519 kB 00:00 2024-09-21T09:40:14.679 INFO:teuthology.orchestra.run.smithi033.stdout:(73/114): python3-pytz-2021.1-5.el9.noarch.rpm 469 kB/s | 51 kB 00:00 2024-09-21T09:40:14.688 INFO:teuthology.orchestra.run.smithi105.stdout:(66/114): python3-packaging-20.9-5.el9.noarch.r 117 kB/s | 77 kB 00:00 2024-09-21T09:40:14.704 INFO:teuthology.orchestra.run.smithi033.stdout:(74/114): python3-requests-oauthlib-1.3.0-12.el 493 kB/s | 54 kB 00:00 2024-09-21T09:40:14.788 INFO:teuthology.orchestra.run.smithi033.stdout:(75/114): python3-toml-0.10.2-6.el9.noarch.rpm 384 kB/s | 42 kB 00:00 2024-09-21T09:40:14.813 INFO:teuthology.orchestra.run.smithi105.stdout:(67/114): python3-pluggy-0.13.1-7.el9.noarch.rp 72 kB/s | 41 kB 00:00 2024-09-21T09:40:14.821 INFO:teuthology.orchestra.run.smithi033.stdout:(76/114): socat-1.7.4.1-6.el9.x86_64.rpm 2.5 MB/s | 304 kB 00:00 2024-09-21T09:40:14.830 INFO:teuthology.orchestra.run.smithi105.stdout:(68/114): python3-prettytable-0.7.2-27.el9.noar 75 kB/s | 42 kB 00:00 2024-09-21T09:40:14.871 INFO:teuthology.orchestra.run.smithi033.stdout:(77/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 6.0 MB/s | 308 kB 00:00 2024-09-21T09:40:14.897 INFO:teuthology.orchestra.run.smithi033.stdout:(78/114): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-21T09:40:14.922 INFO:teuthology.orchestra.run.smithi033.stdout:(79/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 475 kB/s | 64 kB 00:00 2024-09-21T09:40:14.947 INFO:teuthology.orchestra.run.smithi033.stdout:(80/114): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-21T09:40:14.981 INFO:teuthology.orchestra.run.smithi033.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.7 MB/s | 43 kB 00:00 2024-09-21T09:40:15.006 INFO:teuthology.orchestra.run.smithi033.stdout:(82/114): python3-cachetools-4.2.4-1.el9.noarch 1.4 MB/s | 32 kB 00:00 2024-09-21T09:40:15.031 INFO:teuthology.orchestra.run.smithi033.stdout:(83/114): python3-certifi-2023.05.07-4.el9.noar 567 kB/s | 14 kB 00:00 2024-09-21T09:40:15.065 INFO:teuthology.orchestra.run.smithi033.stdout:(84/114): python3-cheroot-8.6.0-4.el9.noarch.rp 5.1 MB/s | 172 kB 00:00 2024-09-21T09:40:15.099 INFO:teuthology.orchestra.run.smithi033.stdout:(85/114): python3-asyncssh-2.13.2-5.el9.noarch. 3.0 MB/s | 548 kB 00:00 2024-09-21T09:40:15.124 INFO:teuthology.orchestra.run.smithi033.stdout:(86/114): python3-cherrypy-18.6.1-2.el9.noarch. 5.9 MB/s | 358 kB 00:00 2024-09-21T09:40:15.150 INFO:teuthology.orchestra.run.smithi033.stdout:(87/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 426 kB/s | 11 kB 00:00 2024-09-21T09:40:15.175 INFO:teuthology.orchestra.run.smithi033.stdout:(88/114): python3-google-auth-2.34.0-1.el9.noar 2.8 MB/s | 220 kB 00:00 2024-09-21T09:40:15.200 INFO:teuthology.orchestra.run.smithi033.stdout:(89/114): python3-jaraco-classes-3.2.1-5.el9.no 352 kB/s | 18 kB 00:00 2024-09-21T09:40:15.225 INFO:teuthology.orchestra.run.smithi033.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 461 kB/s | 23 kB 00:00 2024-09-21T09:40:15.251 INFO:teuthology.orchestra.run.smithi033.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 386 kB/s | 19 kB 00:00 2024-09-21T09:40:15.276 INFO:teuthology.orchestra.run.smithi033.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 392 kB/s | 20 kB 00:00 2024-09-21T09:40:15.301 INFO:teuthology.orchestra.run.smithi033.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 180 kB/s | 9.0 kB 00:00 2024-09-21T09:40:15.327 INFO:teuthology.orchestra.run.smithi033.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 812 kB/s | 41 kB 00:00 2024-09-21T09:40:15.352 INFO:teuthology.orchestra.run.smithi033.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 1.8 MB/s | 46 kB 00:00 2024-09-21T09:40:15.377 INFO:teuthology.orchestra.run.smithi033.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 3.1 MB/s | 79 kB 00:00 2024-09-21T09:40:15.403 INFO:teuthology.orchestra.run.smithi033.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 2.3 MB/s | 58 kB 00:00 2024-09-21T09:40:15.436 INFO:teuthology.orchestra.run.smithi033.stdout:(98/114): python3-pecan-1.4.2-3.el9.noarch.rpm 8.1 MB/s | 272 kB 00:00 2024-09-21T09:40:15.461 INFO:teuthology.orchestra.run.smithi033.stdout:(99/114): python3-portend-3.1.0-2.el9.noarch.rp 657 kB/s | 16 kB 00:00 2024-09-21T09:40:15.487 INFO:teuthology.orchestra.run.smithi033.stdout:(100/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 3.5 MB/s | 90 kB 00:00 2024-09-21T09:40:15.512 INFO:teuthology.orchestra.run.smithi033.stdout:(101/114): python3-repoze-lru-0.7-16.el9.noarch 1.2 MB/s | 31 kB 00:00 2024-09-21T09:40:15.522 INFO:teuthology.orchestra.run.smithi105.stdout:(69/114): python3-pyasn1-0.4.8-6.el9.noarch.rpm 224 kB/s | 159 kB 00:00 2024-09-21T09:40:15.546 INFO:teuthology.orchestra.run.smithi033.stdout:(102/114): python3-kubernetes-26.1.0-2.el9.noar 4.2 MB/s | 1.0 MB 00:00 2024-09-21T09:40:15.571 INFO:teuthology.orchestra.run.smithi033.stdout:(103/114): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-09-21T09:40:15.581 INFO:teuthology.orchestra.run.smithi105.stdout:(70/114): python3-py-1.10.0-6.el9.noarch.rpm 535 kB/s | 477 kB 00:00 2024-09-21T09:40:15.596 INFO:teuthology.orchestra.run.smithi033.stdout:(104/114): python3-routes-2.5.1-5.el9.noarch.rp 2.2 MB/s | 188 kB 00:00 2024-09-21T09:40:15.615 INFO:teuthology.orchestra.run.smithi105.stdout:(71/114): python3-pyasn1-modules-0.4.8-6.el9.no 356 kB/s | 279 kB 00:00 2024-09-21T09:40:15.621 INFO:teuthology.orchestra.run.smithi033.stdout:(105/114): python3-tempora-5.0.0-2.el9.noarch.r 711 kB/s | 36 kB 00:00 2024-09-21T09:40:15.699 INFO:teuthology.orchestra.run.smithi033.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 729 kB/s | 74 kB 00:00 2024-09-21T09:40:15.739 INFO:teuthology.orchestra.run.smithi033.stdout:(107/114): python3-websocket-client-1.2.3-2.el9 2.2 MB/s | 90 kB 00:00 2024-09-21T09:40:15.764 INFO:teuthology.orchestra.run.smithi033.stdout:(108/114): python3-webob-1.8.8-2.el9.noarch.rpm 1.6 MB/s | 230 kB 00:00 2024-09-21T09:40:15.790 INFO:teuthology.orchestra.run.smithi033.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 806 kB/s | 20 kB 00:00 2024-09-21T09:40:15.823 INFO:teuthology.orchestra.run.smithi033.stdout:(110/114): python3-werkzeug-2.0.3-3.el9.1.noarc 5.0 MB/s | 427 kB 00:00 2024-09-21T09:40:16.015 INFO:teuthology.orchestra.run.smithi033.stdout:(111/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 12 MB/s | 16 MB 00:01 2024-09-21T09:40:16.057 INFO:teuthology.orchestra.run.smithi033.stdout:(112/114): thrift-0.15.0-2.el9.x86_64.rpm 5.9 MB/s | 1.6 MB 00:00 2024-09-21T09:40:16.166 INFO:teuthology.orchestra.run.smithi033.stdout:(113/114): librados2-17.2.7-1649.gfbd1a6ab.el9. 9.0 MB/s | 3.1 MB 00:00 2024-09-21T09:40:16.173 INFO:teuthology.orchestra.run.smithi105.stdout:(72/114): python3-pytz-2021.1-5.el9.noarch.rpm 86 kB/s | 51 kB 00:00 2024-09-21T09:40:16.206 INFO:teuthology.orchestra.run.smithi105.stdout:(73/114): python3-requests-oauthlib-1.3.0-12.el 91 kB/s | 54 kB 00:00 2024-09-21T09:40:16.308 INFO:teuthology.orchestra.run.smithi033.stdout:(114/114): librbd1-17.2.7-1649.gfbd1a6ab.el9.x8 10 MB/s | 2.9 MB 00:00 2024-09-21T09:40:16.313 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:16.313 INFO:teuthology.orchestra.run.smithi033.stdout:Total 23 MB/s | 190 MB 00:08 2024-09-21T09:40:16.457 INFO:teuthology.orchestra.run.smithi105.stdout:(74/114): python3-pytest-6.2.2-7.el9.noarch.rpm 556 kB/s | 519 kB 00:00 2024-09-21T09:40:16.766 INFO:teuthology.orchestra.run.smithi105.stdout:(75/114): python3-toml-0.10.2-6.el9.noarch.rpm 75 kB/s | 42 kB 00:00 2024-09-21T09:40:17.064 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction check 2024-09-21T09:40:17.168 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction check succeeded. 2024-09-21T09:40:17.168 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction test 2024-09-21T09:40:17.249 INFO:teuthology.orchestra.run.smithi105.stdout:(76/114): socat-1.7.4.1-6.el9.x86_64.rpm 384 kB/s | 304 kB 00:00 2024-09-21T09:40:17.299 INFO:teuthology.orchestra.run.smithi105.stdout:(77/114): gperftools-libs-2.9.1-3.el9.x86_64.rp 6.0 MB/s | 308 kB 00:00 2024-09-21T09:40:17.325 INFO:teuthology.orchestra.run.smithi105.stdout:(78/114): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-21T09:40:17.349 INFO:teuthology.orchestra.run.smithi105.stdout:(79/114): libunwind-1.6.2-1.el9.x86_64.rpm 2.7 MB/s | 67 kB 00:00 2024-09-21T09:40:17.383 INFO:teuthology.orchestra.run.smithi105.stdout:(80/114): python3-asyncssh-2.13.2-5.el9.noarch. 16 MB/s | 548 kB 00:00 2024-09-21T09:40:17.417 INFO:teuthology.orchestra.run.smithi105.stdout:(81/114): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.3 MB/s | 43 kB 00:00 2024-09-21T09:40:17.451 INFO:teuthology.orchestra.run.smithi105.stdout:(82/114): xmlstarlet-1.6.1-20.el9.x86_64.rpm 93 kB/s | 64 kB 00:00 2024-09-21T09:40:17.476 INFO:teuthology.orchestra.run.smithi105.stdout:(83/114): python3-cachetools-4.2.4-1.el9.noarch 560 kB/s | 32 kB 00:00 2024-09-21T09:40:17.518 INFO:teuthology.orchestra.run.smithi105.stdout:(84/114): python3-cheroot-8.6.0-4.el9.noarch.rp 4.9 MB/s | 172 kB 00:00 2024-09-21T09:40:17.543 INFO:teuthology.orchestra.run.smithi105.stdout:(85/114): python3-certifi-2023.05.07-4.el9.noar 153 kB/s | 14 kB 00:00 2024-09-21T09:40:17.577 INFO:teuthology.orchestra.run.smithi105.stdout:(86/114): python3-cherrypy-18.6.1-2.el9.noarch. 6.2 MB/s | 358 kB 00:00 2024-09-21T09:40:17.603 INFO:teuthology.orchestra.run.smithi105.stdout:(87/114): python3-google-auth-2.34.0-1.el9.noar 3.7 MB/s | 220 kB 00:00 2024-09-21T09:40:17.628 INFO:teuthology.orchestra.run.smithi105.stdout:(88/114): python3-jaraco-8.2.1-3.el9.noarch.rpm 213 kB/s | 11 kB 00:00 2024-09-21T09:40:17.653 INFO:teuthology.orchestra.run.smithi105.stdout:(89/114): python3-jaraco-classes-3.2.1-5.el9.no 352 kB/s | 18 kB 00:00 2024-09-21T09:40:17.678 INFO:teuthology.orchestra.run.smithi105.stdout:(90/114): python3-jaraco-collections-3.0.0-8.el 461 kB/s | 23 kB 00:00 2024-09-21T09:40:17.704 INFO:teuthology.orchestra.run.smithi105.stdout:(91/114): python3-jaraco-functools-3.5.0-2.el9. 387 kB/s | 19 kB 00:00 2024-09-21T09:40:17.729 INFO:teuthology.orchestra.run.smithi105.stdout:(92/114): python3-jaraco-text-3.2.0-6.el9.noarc 392 kB/s | 20 kB 00:00 2024-09-21T09:40:17.754 INFO:teuthology.orchestra.run.smithi105.stdout:(93/114): python3-jwt+crypto-2.4.0-1.el9.noarch 180 kB/s | 9.0 kB 00:00 2024-09-21T09:40:17.779 INFO:teuthology.orchestra.run.smithi105.stdout:(94/114): python3-jwt-2.4.0-1.el9.noarch.rpm 816 kB/s | 41 kB 00:00 2024-09-21T09:40:17.906 INFO:teuthology.orchestra.run.smithi105.stdout:(95/114): python3-logutils-0.3.5-21.el9.noarch. 1.8 MB/s | 46 kB 00:00 2024-09-21T09:40:17.906 INFO:teuthology.orchestra.run.smithi105.stdout:(96/114): python3-more-itertools-8.12.0-2.el9.n 3.1 MB/s | 79 kB 00:00 2024-09-21T09:40:17.906 INFO:teuthology.orchestra.run.smithi105.stdout:(97/114): python3-natsort-7.1.1-5.el9.noarch.rp 2.3 MB/s | 58 kB 00:00 2024-09-21T09:40:17.906 INFO:teuthology.orchestra.run.smithi105.stdout:(98/114): python3-pecan-1.4.2-3.el9.noarch.rpm 11 MB/s | 272 kB 00:00 2024-09-21T09:40:17.914 INFO:teuthology.orchestra.run.smithi105.stdout:(99/114): python3-kubernetes-26.1.0-2.el9.noarc 6.4 MB/s | 1.0 MB 00:00 2024-09-21T09:40:17.939 INFO:teuthology.orchestra.run.smithi105.stdout:(100/114): python3-portend-3.1.0-2.el9.noarch.r 280 kB/s | 16 kB 00:00 2024-09-21T09:40:17.965 INFO:teuthology.orchestra.run.smithi105.stdout:(101/114): python3-repoze-lru-0.7-16.el9.noarch 1.2 MB/s | 31 kB 00:00 2024-09-21T09:40:17.990 INFO:teuthology.orchestra.run.smithi105.stdout:(102/114): python3-pyOpenSSL-21.0.0-1.el9.noarc 1.2 MB/s | 90 kB 00:00 2024-09-21T09:40:18.015 INFO:teuthology.orchestra.run.smithi105.stdout:(103/114): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2024-09-21T09:40:18.041 INFO:teuthology.orchestra.run.smithi105.stdout:(104/114): python3-routes-2.5.1-5.el9.noarch.rp 2.4 MB/s | 188 kB 00:00 2024-09-21T09:40:18.066 INFO:teuthology.orchestra.run.smithi105.stdout:(105/114): python3-tempora-5.0.0-2.el9.noarch.r 709 kB/s | 36 kB 00:00 2024-09-21T09:40:18.092 INFO:teuthology.orchestra.run.smithi105.stdout:(106/114): python3-typing-extensions-4.12.2-2.e 1.4 MB/s | 74 kB 00:00 2024-09-21T09:40:18.117 INFO:teuthology.orchestra.run.smithi105.stdout:(107/114): python3-websocket-client-1.2.3-2.el9 3.5 MB/s | 90 kB 00:00 2024-09-21T09:40:18.150 INFO:teuthology.orchestra.run.smithi105.stdout:(108/114): python3-webob-1.8.8-2.el9.noarch.rpm 2.7 MB/s | 230 kB 00:00 2024-09-21T09:40:18.176 INFO:teuthology.orchestra.run.smithi105.stdout:(109/114): python3-zc-lockfile-2.0-10.el9.noarc 797 kB/s | 20 kB 00:00 2024-09-21T09:40:18.209 INFO:teuthology.orchestra.run.smithi105.stdout:(110/114): python3-werkzeug-2.0.3-3.el9.1.noarc 4.5 MB/s | 427 kB 00:00 2024-09-21T09:40:18.369 INFO:teuthology.orchestra.run.smithi105.stdout:(111/114): python3-scipy-1.6.2-8.el9.x86_64.rpm 7.4 MB/s | 16 MB 00:02 2024-09-21T09:40:18.418 INFO:teuthology.orchestra.run.smithi105.stdout:(112/114): thrift-0.15.0-2.el9.x86_64.rpm 6.5 MB/s | 1.6 MB 00:00 2024-09-21T09:40:18.519 INFO:teuthology.orchestra.run.smithi105.stdout:(113/114): librados2-17.2.7-1649.gfbd1a6ab.el9. 10 MB/s | 3.1 MB 00:00 2024-09-21T09:40:18.913 INFO:teuthology.orchestra.run.smithi033.stdout:Transaction test succeeded. 2024-09-21T09:40:18.913 INFO:teuthology.orchestra.run.smithi033.stdout:Running transaction 2024-09-21T09:40:18.961 INFO:teuthology.orchestra.run.smithi105.stdout:(114/114): librbd1-17.2.7-1649.gfbd1a6ab.el9.x8 5.0 MB/s | 2.9 MB 00:00 2024-09-21T09:40:18.965 INFO:teuthology.orchestra.run.smithi105.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:40:18.966 INFO:teuthology.orchestra.run.smithi105.stdout:Total 13 MB/s | 190 MB 00:15 2024-09-21T09:40:19.687 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction check 2024-09-21T09:40:19.782 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction check succeeded. 2024-09-21T09:40:19.782 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction test 2024-09-21T09:40:21.431 INFO:teuthology.orchestra.run.smithi105.stdout:Transaction test succeeded. 2024-09-21T09:40:21.431 INFO:teuthology.orchestra.run.smithi105.stdout:Running transaction 2024-09-21T09:40:23.727 INFO:teuthology.orchestra.run.smithi033.stdout: Preparing : 1/1 2024-09-21T09:40:23.802 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-21T09:40:23.836 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-21T09:40:23.899 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-21T09:40:24.227 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-21T09:40:24.328 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/116 2024-09-21T09:40:24.444 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/116 2024-09-21T09:40:24.479 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:40:24.555 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:40:24.614 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/116 2024-09-21T09:40:24.645 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-21T09:40:25.104 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-21T09:40:25.140 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:40:25.215 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:40:25.241 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:40:25.342 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:40:25.367 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 12/116 2024-09-21T09:40:25.498 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 12/116 2024-09-21T09:40:25.530 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-21T09:40:25.562 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-21T09:40:25.609 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-21T09:40:25.650 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-21T09:40:25.725 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-21T09:40:25.785 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-21T09:40:25.824 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-21T09:40:25.851 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-21T09:40:25.924 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-21T09:40:25.983 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-21T09:40:26.023 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-21T09:40:26.065 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-21T09:40:26.091 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-21T09:40:26.132 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el 26/116 2024-09-21T09:40:26.208 INFO:teuthology.orchestra.run.smithi105.stdout: Preparing : 1/1 2024-09-21T09:40:26.213 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 27/116 2024-09-21T09:40:26.288 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : thrift-0.15.0-2.el9.x86_64 1/116 2024-09-21T09:40:26.314 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 2/116 2024-09-21T09:40:26.370 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-21T09:40:26.378 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 3/116 2024-09-21T09:40:26.429 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-21T09:40:26.680 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 4/116 2024-09-21T09:40:26.773 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/116 2024-09-21T09:40:26.887 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 5/116 2024-09-21T09:40:26.916 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:40:26.998 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-21T09:40:27.008 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:40:27.061 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/116 2024-09-21T09:40:27.096 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 8/116 2024-09-21T09:40:27.136 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-21T09:40:27.179 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-21T09:40:27.213 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-21T09:40:27.239 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-21T09:40:27.279 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-21T09:40:27.314 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 36/116 2024-09-21T09:40:27.353 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 37/116 2024-09-21T09:40:27.382 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-21T09:40:27.419 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-21T09:40:27.476 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-21T09:40:27.537 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 9/116 2024-09-21T09:40:27.550 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-21T09:40:27.568 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:40:27.586 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-21T09:40:27.623 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-21T09:40:27.643 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:40:27.662 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-21T09:40:27.669 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:40:27.707 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-21T09:40:27.749 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-21T09:40:27.769 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:40:27.795 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 12/116 2024-09-21T09:40:27.895 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-21T09:40:27.922 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 12/116 2024-09-21T09:40:27.936 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-21T09:40:27.958 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 13/116 2024-09-21T09:40:27.979 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-21T09:40:27.990 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : liboath-2.6.7-2.el9.x86_64 14/116 2024-09-21T09:40:28.026 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 15/116 2024-09-21T09:40:28.071 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 16/116 2024-09-21T09:40:28.088 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-21T09:40:28.134 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 17/116 2024-09-21T09:40:28.137 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-21T09:40:28.178 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-21T09:40:28.179 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 18/116 2024-09-21T09:40:28.214 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-21T09:40:28.219 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-packaging-20.9-5.el9.noarch 19/116 2024-09-21T09:40:28.246 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 20/116 2024-09-21T09:40:28.259 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-21T09:40:28.324 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 21/116 2024-09-21T09:40:28.385 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 22/116 2024-09-21T09:40:28.391 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-21T09:40:28.424 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 23/116 2024-09-21T09:40:28.425 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-21T09:40:28.465 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-21T09:40:28.467 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 24/116 2024-09-21T09:40:28.503 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 25/116 2024-09-21T09:40:28.551 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el 26/116 2024-09-21T09:40:28.578 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-21T09:40:28.630 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 27/116 2024-09-21T09:40:28.636 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-21T09:40:28.772 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 28/116 2024-09-21T09:40:28.832 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 29/116 2024-09-21T09:40:29.377 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rsa-4.9-2.el9.noarch 30/116 2024-09-21T09:40:29.409 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-21T09:40:29.457 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-21T09:40:29.491 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-21T09:40:29.524 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-21T09:40:29.562 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 31/116 2024-09-21T09:40:29.646 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-21T09:40:29.691 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 32/116 2024-09-21T09:40:29.700 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-21T09:40:29.733 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 33/116 2024-09-21T09:40:29.733 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-21T09:40:29.768 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 34/116 2024-09-21T09:40:29.807 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 35/116 2024-09-21T09:40:29.843 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 36/116 2024-09-21T09:40:29.881 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 37/116 2024-09-21T09:40:29.911 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 38/116 2024-09-21T09:40:29.926 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-21T09:40:29.947 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 39/116 2024-09-21T09:40:29.965 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-21T09:40:30.003 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 40/116 2024-09-21T09:40:30.045 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-21T09:40:30.077 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 41/116 2024-09-21T09:40:30.116 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 42/116 2024-09-21T09:40:30.152 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 43/116 2024-09-21T09:40:30.208 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 44/116 2024-09-21T09:40:30.261 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 45/116 2024-09-21T09:40:30.303 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 46/116 2024-09-21T09:40:30.444 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 47/116 2024-09-21T09:40:30.491 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 48/116 2024-09-21T09:40:30.533 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 49/116 2024-09-21T09:40:30.637 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 50/116 2024-09-21T09:40:30.684 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 51/116 2024-09-21T09:40:30.724 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 52/116 2024-09-21T09:40:30.747 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-21T09:40:30.761 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 53/116 2024-09-21T09:40:30.814 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 54/116 2024-09-21T09:40:30.868 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-21T09:40:30.947 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 55/116 2024-09-21T09:40:30.980 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-py-1.10.0-6.el9.noarch 56/116 2024-09-21T09:40:31.020 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 57/116 2024-09-21T09:40:31.135 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pluggy-0.13.1-7.el9.noarch 58/116 2024-09-21T09:40:31.199 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 59/116 2024-09-21T09:40:31.936 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 60/116 2024-09-21T09:40:31.979 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 61/116 2024-09-21T09:40:32.012 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 62/116 2024-09-21T09:40:32.046 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 63/116 2024-09-21T09:40:32.148 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-iniconfig-1.1.1-7.el9.noarch 64/116 2024-09-21T09:40:32.213 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 65/116 2024-09-21T09:40:32.241 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-21T09:40:32.256 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-attrs-20.3.0-7.el9.noarch 66/116 2024-09-21T09:40:32.312 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-21T09:40:32.438 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : openblas-0.3.26-2.el9.x86_64 67/116 2024-09-21T09:40:32.481 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 68/116 2024-09-21T09:40:32.558 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 69/116 2024-09-21T09:40:32.620 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-21T09:40:32.657 INFO:teuthology.orchestra.run.smithi033.stdout: Upgrading : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 75/116 2024-09-21T09:40:32.748 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 75/116 2024-09-21T09:40:32.791 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 76/116 2024-09-21T09:40:32.849 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-21T09:40:32.913 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-21T09:40:33.111 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-21T09:40:33.152 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-21T09:40:33.229 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 70/116 2024-09-21T09:40:33.233 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-21T09:40:33.333 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-21T09:40:33.346 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 71/116 2024-09-21T09:40:33.477 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-21T09:40:33.540 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-21T09:40:33.568 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-21T09:40:33.607 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-21T09:40:33.643 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-21T09:40:33.670 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-21T09:40:34.125 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 89/116 2024-09-21T09:40:34.151 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 89/116 2024-09-21T09:40:34.181 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.e 90/116 2024-09-21T09:40:34.258 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab. 91/116 2024-09-21T09:40:34.284 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9. 92/116 2024-09-21T09:40:34.622 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 72/116 2024-09-21T09:40:34.697 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 73/116 2024-09-21T09:40:34.996 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 74/116 2024-09-21T09:40:35.116 INFO:teuthology.orchestra.run.smithi105.stdout: Upgrading : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 75/116 2024-09-21T09:40:35.224 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 75/116 2024-09-21T09:40:35.264 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 76/116 2024-09-21T09:40:35.322 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-ply-3.11-14.el9.noarch 77/116 2024-09-21T09:40:35.385 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 78/116 2024-09-21T09:40:35.573 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 79/116 2024-09-21T09:40:35.618 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 80/116 2024-09-21T09:40:35.706 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 81/116 2024-09-21T09:40:35.803 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 82/116 2024-09-21T09:40:35.953 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 83/116 2024-09-21T09:40:35.997 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 84/116 2024-09-21T09:40:36.035 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 85/116 2024-09-21T09:40:36.072 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 86/116 2024-09-21T09:40:36.109 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 87/116 2024-09-21T09:40:36.137 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : fuse-2.9.9-16.el9.x86_64 88/116 2024-09-21T09:40:36.547 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 89/116 2024-09-21T09:40:36.601 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 89/116 2024-09-21T09:40:36.639 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.e 90/116 2024-09-21T09:40:36.713 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab. 91/116 2024-09-21T09:40:36.742 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9. 92/116 2024-09-21T09:40:36.830 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:40:36.863 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:40:37.577 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:40:37.608 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 94/116 2024-09-21T09:40:37.691 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 94/116 2024-09-21T09:40:37.691 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:40:37.691 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:40:37.691 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:37.734 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 95/116 2024-09-21T09:40:39.161 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:40:39.205 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:40:39.730 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 93/116 2024-09-21T09:40:39.775 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 94/116 2024-09-21T09:40:39.863 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 94/116 2024-09-21T09:40:39.863 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-21T09:40:39.863 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-21T09:40:39.864 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:39.901 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 95/116 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 95/116 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /sys 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /proc 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /mnt 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /var/tmp 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /home 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /root 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout:skipping the directory /tmp 2024-09-21T09:40:49.778 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:49.843 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 96/116 2024-09-21T09:40:50.889 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 96/116 2024-09-21T09:40:50.921 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 97/116 2024-09-21T09:40:51.397 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 95/116 2024-09-21T09:40:51.398 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /sys 2024-09-21T09:40:51.398 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /proc 2024-09-21T09:40:51.398 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /mnt 2024-09-21T09:40:51.398 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /var/tmp 2024-09-21T09:40:51.398 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /home 2024-09-21T09:40:51.398 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /root 2024-09-21T09:40:51.398 INFO:teuthology.orchestra.run.smithi105.stdout:skipping the directory /tmp 2024-09-21T09:40:51.398 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:51.470 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 96/116 2024-09-21T09:40:51.961 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 97/116 2024-09-21T09:40:51.991 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 98/116 2024-09-21T09:40:52.096 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 98/116 2024-09-21T09:40:52.270 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el 99/116 2024-09-21T09:40:52.292 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 100/116 2024-09-21T09:40:52.316 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 100/116 2024-09-21T09:40:52.317 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:52.317 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:40:52.317 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:40:52.317 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:40:52.317 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:52.368 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 101/116 2024-09-21T09:40:52.444 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 96/116 2024-09-21T09:40:52.490 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 97/116 2024-09-21T09:40:52.560 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 101/116 2024-09-21T09:40:52.585 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 102/116 2024-09-21T09:40:52.610 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 102/116 2024-09-21T09:40:52.610 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:52.610 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:40:52.610 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:40:52.610 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:40:52.610 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:52.994 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 103/116 2024-09-21T09:40:53.019 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 103/116 2024-09-21T09:40:53.019 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:53.019 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:40:53.019 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:40:53.020 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:40:53.020 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:53.488 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 97/116 2024-09-21T09:40:53.525 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 98/116 2024-09-21T09:40:53.624 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 98/116 2024-09-21T09:40:53.794 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el 99/116 2024-09-21T09:40:53.819 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 100/116 2024-09-21T09:40:53.843 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 100/116 2024-09-21T09:40:53.843 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:53.843 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-21T09:40:53.843 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:40:53.843 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-21T09:40:53.843 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:53.886 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 101/116 2024-09-21T09:40:54.068 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 101/116 2024-09-21T09:40:54.104 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 102/116 2024-09-21T09:40:54.127 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 102/116 2024-09-21T09:40:54.127 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:54.127 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-21T09:40:54.127 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:40:54.127 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-21T09:40:54.127 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:54.403 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 104/116 2024-09-21T09:40:54.432 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 104/116 2024-09-21T09:40:54.432 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:54.432 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:40:54.432 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:40:54.432 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:40:54.432 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:54.496 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 103/116 2024-09-21T09:40:54.519 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 103/116 2024-09-21T09:40:54.520 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:54.520 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-21T09:40:54.520 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:40:54.520 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-21T09:40:54.520 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:55.756 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 105/116 2024-09-21T09:40:55.788 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 106/116 2024-09-21T09:40:55.839 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 104/116 2024-09-21T09:40:55.865 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 104/116 2024-09-21T09:40:55.865 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:55.865 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-21T09:40:55.865 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:40:55.865 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-21T09:40:55.865 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:55.867 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 106/116 2024-09-21T09:40:55.868 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:55.868 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:40:55.868 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:40:55.868 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:40:55.868 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:55.906 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 107/116 2024-09-21T09:40:55.930 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 107/116 2024-09-21T09:40:55.930 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:55.930 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:40:55.930 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:56.240 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 108/116 2024-09-21T09:40:56.265 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 108/116 2024-09-21T09:40:56.265 INFO:teuthology.orchestra.run.smithi033.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:56.265 INFO:teuthology.orchestra.run.smithi033.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:40:56.265 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:40:56.265 INFO:teuthology.orchestra.run.smithi033.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:40:56.265 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:40:57.118 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 105/116 2024-09-21T09:40:57.148 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 106/116 2024-09-21T09:40:57.227 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 106/116 2024-09-21T09:40:57.227 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:57.227 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-21T09:40:57.227 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:40:57.227 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-21T09:40:57.227 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:57.266 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 107/116 2024-09-21T09:40:57.288 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 107/116 2024-09-21T09:40:57.288 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:57.289 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-21T09:40:57.289 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:40:57.584 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 108/116 2024-09-21T09:40:57.607 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 108/116 2024-09-21T09:40:57.607 INFO:teuthology.orchestra.run.smithi105.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-21T09:40:57.607 INFO:teuthology.orchestra.run.smithi105.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-21T09:40:57.607 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:40:57.607 INFO:teuthology.orchestra.run.smithi105.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-21T09:40:57.607 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:41:00.967 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 109/116 2024-09-21T09:41:01.001 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 110/116 2024-09-21T09:41:01.043 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 111/116 2024-09-21T09:41:01.171 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 112/116 2024-09-21T09:41:01.208 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-21T09:41:01.231 INFO:teuthology.orchestra.run.smithi033.stdout: Installing : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_ 114/116 2024-09-21T09:41:01.231 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-21T09:41:01.340 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-21T09:41:01.340 INFO:teuthology.orchestra.run.smithi033.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:41:02.009 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 109/116 2024-09-21T09:41:02.053 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 110/116 2024-09-21T09:41:02.094 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 111/116 2024-09-21T09:41:02.213 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 112/116 2024-09-21T09:41:02.252 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : python3-pytest-6.2.2-7.el9.noarch 113/116 2024-09-21T09:41:02.267 INFO:teuthology.orchestra.run.smithi105.stdout: Installing : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_ 114/116 2024-09-21T09:41:02.267 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-21T09:41:02.351 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 115/116 2024-09-21T09:41:02.351 INFO:teuthology.orchestra.run.smithi105.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:41:06.368 INFO:teuthology.orchestra.run.smithi033.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:41:06.368 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 5/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 8/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 9/116 2024-09-21T09:41:06.369 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 12/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_ 13/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 14/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 15/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 16/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 17/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 18/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el 19/116 2024-09-21T09:41:06.370 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9. 20/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 21/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 22/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 24/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 25/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 26/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 27/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab. 28/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 29/116 2024-09-21T09:41:06.371 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 30/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 31/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el 32/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 33/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.e 34/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 35/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-21T09:41:06.382 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-21T09:41:06.383 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-21T09:41:06.384 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-21T09:41:06.385 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-21T09:41:06.386 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 113/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-21T09:41:06.387 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 115/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 1/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 3/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 4/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a 5/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 6/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 7/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 8/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 9/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 10/116 2024-09-21T09:41:07.468 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 11/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 12/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_ 13/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 14/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 15/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 16/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86 17/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 18/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el 19/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9. 20/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_6 21/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 22/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 23/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 24/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 25/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 26/116 2024-09-21T09:41:07.469 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 27/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab. 28/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noa 29/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.n 30/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd 31/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el 32/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 33/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.e 34/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 35/116 2024-09-21T09:41:07.476 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 36/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 37/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 38/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 39/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 40/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-ply-3.11-14.el9.noarch 41/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 42/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 43/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 44/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 45/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 46/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 47/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 48/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 49/116 2024-09-21T09:41:07.477 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 50/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 51/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 52/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 53/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-attrs-20.3.0-7.el9.noarch 54/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 55/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 56/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-iniconfig-1.1.1-7.el9.noarch 57/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 58/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 59/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 60/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 61/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 62/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 63/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 64/116 2024-09-21T09:41:07.478 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 65/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 66/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pluggy-0.13.1-7.el9.noarch 67/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 68/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-py-1.10.0-6.el9.noarch 69/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 70/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 71/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pytest-6.2.2-7.el9.noarch 72/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 73/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 74/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 75/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 76/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 77/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 78/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 79/116 2024-09-21T09:41:07.479 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 80/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 81/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 82/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 83/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 84/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 85/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 86/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 87/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 88/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 89/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 90/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 91/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 92/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 93/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 94/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 95/116 2024-09-21T09:41:07.480 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 96/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 97/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 98/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 99/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 100/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 101/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 102/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 103/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 104/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 105/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 106/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 107/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 108/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 109/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 110/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 111/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 112/116 2024-09-21T09:41:07.481 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 113/116 2024-09-21T09:41:07.482 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 114/116 2024-09-21T09:41:07.482 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 115/116 2024-09-21T09:41:08.355 INFO:teuthology.orchestra.run.smithi033.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:41:08.355 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:41:08.355 INFO:teuthology.orchestra.run.smithi033.stdout:Upgraded: 2024-09-21T09:41:08.355 INFO:teuthology.orchestra.run.smithi033.stdout: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout:Installed: 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:08.356 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-21T09:41:08.357 INFO:teuthology.orchestra.run.smithi033.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:41:08.358 INFO:teuthology.orchestra.run.smithi033.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:41:08.359 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-21T09:41:08.360 INFO:teuthology.orchestra.run.smithi033.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:41:08.361 INFO:teuthology.orchestra.run.smithi033.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.362 INFO:teuthology.orchestra.run.smithi033.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.363 INFO:teuthology.orchestra.run.smithi033.stdout: rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:08.364 INFO:teuthology.orchestra.run.smithi033.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:41:08.364 INFO:teuthology.orchestra.run.smithi033.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:41:08.364 INFO:teuthology.orchestra.run.smithi033.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:41:08.364 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:41:08.364 INFO:teuthology.orchestra.run.smithi033.stdout:Complete! 2024-09-21T09:41:08.692 DEBUG:teuthology.parallel:result is None 2024-09-21T09:41:09.533 INFO:teuthology.orchestra.run.smithi105.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 116/116 2024-09-21T09:41:09.533 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:41:09.533 INFO:teuthology.orchestra.run.smithi105.stdout:Upgraded: 2024-09-21T09:41:09.533 INFO:teuthology.orchestra.run.smithi105.stdout: librados2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.533 INFO:teuthology.orchestra.run.smithi105.stdout: librbd1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.533 INFO:teuthology.orchestra.run.smithi105.stdout:Installed: 2024-09-21T09:41:09.533 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-base-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-grafana-dashboards-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-immutable-object-cache-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mds-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-dashboard-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-diskprediction-local-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-modules-core-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mgr-rook-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-mon-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.534 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-osd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-prometheus-alerts-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-radosgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-selinux-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: ceph-test-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: cephadm-2:17.2.7-1649.gfbd1a6ab.el9.noarch 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: fuse-2.9.9-16.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: libcephfs2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: libcephsqlite-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-21T09:41:09.535 INFO:teuthology.orchestra.run.smithi105.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: librados-devel-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: libradosstriper1-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: librgw2-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: python3-attrs-20.3.0-7.el9.noarch 2024-09-21T09:41:09.536 INFO:teuthology.orchestra.run.smithi105.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-argparse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ceph-common-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cephfs-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-iniconfig-1.1.1-7.el9.noarch 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-21T09:41:09.537 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-21T09:41:09.538 INFO:teuthology.orchestra.run.smithi105.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-packaging-20.9-5.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pluggy-0.13.1-7.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-ply-3.11-14.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-py-1.10.0-6.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-21T09:41:09.539 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytest-6.2.2-7.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rados-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rgw-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-21T09:41:09.540 INFO:teuthology.orchestra.run.smithi105.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-fuse-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-mirror-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: rbd-nbd-2:17.2.7-1649.gfbd1a6ab.el9.x86_64 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:41:09.541 INFO:teuthology.orchestra.run.smithi105.stdout:Complete! 2024-09-21T09:41:09.857 DEBUG:teuthology.parallel:result is None 2024-09-21T09:41:09.858 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:41:09.858 INFO:teuthology.packaging:ref: None 2024-09-21T09:41:09.858 INFO:teuthology.packaging:tag: None 2024-09-21T09:41:09.858 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:41:09.858 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:41:09.858 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:41:10.064 DEBUG:teuthology.orchestra.run.smithi033:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:41:10.091 INFO:teuthology.orchestra.run.smithi033.stdout:17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:41:10.091 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:41:10.091 INFO:teuthology.task.install:The correct ceph version 17.2.7-1649.gfbd1a6ab is installed. 2024-09-21T09:41:10.093 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-21T09:41:10.093 INFO:teuthology.packaging:ref: None 2024-09-21T09:41:10.093 INFO:teuthology.packaging:tag: None 2024-09-21T09:41:10.093 INFO:teuthology.packaging:branch: quincy 2024-09-21T09:41:10.093 INFO:teuthology.packaging:sha1: 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:41:10.093 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-09-21T09:41:10.287 DEBUG:teuthology.orchestra.run.smithi105:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-21T09:41:10.310 INFO:teuthology.orchestra.run.smithi105.stdout:17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:41:10.310 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1649.gfbd1a6ab.el9 2024-09-21T09:41:10.310 INFO:teuthology.task.install:The correct ceph version 17.2.7-1649.gfbd1a6ab is installed. 2024-09-21T09:41:10.312 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-21T09:41:10.312 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:41:10.312 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:41:10.345 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:41:10.345 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-21T09:41:10.376 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-21T09:41:10.376 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:41:10.376 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:41:10.415 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:41:10.483 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:41:10.483 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/daemon-helper 2024-09-21T09:41:10.508 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-21T09:41:10.572 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-21T09:41:10.573 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:41:10.573 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:41:10.602 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:41:10.670 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:41:10.670 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-21T09:41:10.695 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-21T09:41:10.760 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-21T09:41:10.760 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:41:10.760 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:41:10.788 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:41:10.855 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:41:10.856 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/usr/bin/stdin-killer 2024-09-21T09:41:10.879 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-21T09:41:10.944 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:41:10.953 INFO:teuthology.task.print:**** done install task... 2024-09-21T09:41:10.953 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:41:10.964 INFO:teuthology.task.print:**** done start installing quincy cephadm ... 2024-09-21T09:41:10.964 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-21T09:41:11.049 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'} 2024-09-21T09:41:11.050 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:41:11.050 INFO:tasks.cephadm:Cluster fsid is a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:11.050 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-21T09:41:11.050 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.33', 'mon.c': '[v2:172.21.15.33:3301,v1:172.21.15.33:6790]', 'mon.b': '172.21.15.105'} 2024-09-21T09:41:11.050 INFO:tasks.cephadm:First mon is mon.a on smithi033 2024-09-21T09:41:11.051 INFO:tasks.cephadm:First mgr is y 2024-09-21T09:41:11.051 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-21T09:41:11.051 DEBUG:teuthology.orchestra.run.smithi033:> sudo hostname $(hostname -s) 2024-09-21T09:41:11.079 DEBUG:teuthology.orchestra.run.smithi105:> sudo hostname $(hostname -s) 2024-09-21T09:41:11.104 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-21T09:41:11.104 DEBUG:teuthology.orchestra.run.smithi033:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:41:11.350 INFO:teuthology.orchestra.run.smithi033.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 21 09:41 /home/ubuntu/cephtest/cephadm 2024-09-21T09:41:11.350 DEBUG:teuthology.orchestra.run.smithi105:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-21T09:41:11.478 INFO:teuthology.orchestra.run.smithi105.stdout:-rw-r--r--. 1 ubuntu ubuntu 378585 Sep 21 09:41 /home/ubuntu/cephtest/cephadm 2024-09-21T09:41:11.478 DEBUG:teuthology.orchestra.run.smithi033:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:41:11.498 DEBUG:teuthology.orchestra.run.smithi105:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-21T09:41:11.524 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-21T09:41:11.524 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-21T09:41:11.540 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-21T09:41:13.186 INFO:teuthology.orchestra.run.smithi105.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:41:13.197 INFO:teuthology.orchestra.run.smithi033.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:41:39.721 INFO:teuthology.orchestra.run.smithi105.stdout:{ 2024-09-21T09:41:39.721 INFO:teuthology.orchestra.run.smithi105.stdout: "ceph_version": "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)", 2024-09-21T09:41:39.721 INFO:teuthology.orchestra.run.smithi105.stdout: "image_id": "fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9", 2024-09-21T09:41:39.721 INFO:teuthology.orchestra.run.smithi105.stdout: "repo_digests": [ 2024-09-21T09:41:39.721 INFO:teuthology.orchestra.run.smithi105.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2" 2024-09-21T09:41:39.721 INFO:teuthology.orchestra.run.smithi105.stdout: ] 2024-09-21T09:41:39.721 INFO:teuthology.orchestra.run.smithi105.stdout:} 2024-09-21T09:41:41.952 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:41:41.953 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph_version": "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)", 2024-09-21T09:41:41.953 INFO:teuthology.orchestra.run.smithi033.stdout: "image_id": "fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9", 2024-09-21T09:41:41.953 INFO:teuthology.orchestra.run.smithi033.stdout: "repo_digests": [ 2024-09-21T09:41:41.953 INFO:teuthology.orchestra.run.smithi033.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2" 2024-09-21T09:41:41.953 INFO:teuthology.orchestra.run.smithi033.stdout: ] 2024-09-21T09:41:41.953 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:41:41.975 DEBUG:teuthology.orchestra.run.smithi033:> sudo mkdir -p /etc/ceph 2024-09-21T09:41:42.004 DEBUG:teuthology.orchestra.run.smithi105:> sudo mkdir -p /etc/ceph 2024-09-21T09:41:42.031 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 777 /etc/ceph 2024-09-21T09:41:42.069 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 777 /etc/ceph 2024-09-21T09:41:42.096 INFO:tasks.cephadm:Writing seed config... 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-21T09:41:42.097 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-21T09:41:42.098 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:41:42.098 DEBUG:teuthology.orchestra.run.smithi033:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-21T09:41:42.124 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = a29e8460-77fd-11ef-baf6-efdc52797490 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-21T09:41:42.125 DEBUG:teuthology.orchestra.run.smithi033:mon.a> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a.service 2024-09-21T09:41:42.167 DEBUG:teuthology.orchestra.run.smithi033:mgr.y> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y.service 2024-09-21T09:41:42.210 INFO:tasks.cephadm:Bootstrapping... 2024-09-21T09:41:42.210 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid a29e8460-77fd-11ef-baf6-efdc52797490 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.33 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:41:42.417 INFO:teuthology.orchestra.run.smithi033.stdout:-------------------------------------------------------------------------------- 2024-09-21T09:41:42.418 INFO:teuthology.orchestra.run.smithi033.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'a29e8460-77fd-11ef-baf6-efdc52797490', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.33', '--skip-admin-label'] 2024-09-21T09:41:42.442 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:41:42.443 INFO:teuthology.orchestra.run.smithi033.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-21T09:41:42.443 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying podman|docker is present... 2024-09-21T09:41:42.466 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:41:42.466 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying lvm2 is present... 2024-09-21T09:41:42.466 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying time synchronization is in place... 2024-09-21T09:41:42.474 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:41:42.474 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:41:42.480 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:41:42.480 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-09-21T09:41:42.487 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout enabled 2024-09-21T09:41:42.494 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout active 2024-09-21T09:41:42.494 INFO:teuthology.orchestra.run.smithi033.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:41:42.495 INFO:teuthology.orchestra.run.smithi033.stdout:Repeating the final host check... 2024-09-21T09:41:42.524 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stdout 5.2.2 2024-09-21T09:41:42.524 INFO:teuthology.orchestra.run.smithi033.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-21T09:41:42.524 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl is present 2024-09-21T09:41:42.524 INFO:teuthology.orchestra.run.smithi033.stdout:lvcreate is present 2024-09-21T09:41:42.531 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-21T09:41:42.531 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-21T09:41:42.538 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-21T09:41:42.538 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-09-21T09:41:42.545 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout enabled 2024-09-21T09:41:42.552 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout active 2024-09-21T09:41:42.552 INFO:teuthology.orchestra.run.smithi033.stdout:Unit chronyd.service is enabled and running 2024-09-21T09:41:42.552 INFO:teuthology.orchestra.run.smithi033.stdout:Host looks OK 2024-09-21T09:41:42.552 INFO:teuthology.orchestra.run.smithi033.stdout:Cluster fsid: a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:42.552 INFO:teuthology.orchestra.run.smithi033.stdout:Acquiring lock 140151984825248 on /run/cephadm/a29e8460-77fd-11ef-baf6-efdc52797490.lock 2024-09-21T09:41:42.552 INFO:teuthology.orchestra.run.smithi033.stdout:Lock 140151984825248 acquired on /run/cephadm/a29e8460-77fd-11ef-baf6-efdc52797490.lock 2024-09-21T09:41:42.553 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying IP 172.21.15.33 port 3300 ... 2024-09-21T09:41:42.553 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying IP 172.21.15.33 port 6789 ... 2024-09-21T09:41:42.553 INFO:teuthology.orchestra.run.smithi033.stdout:Base mon IP(s) is [172.21.15.33:3300, 172.21.15.33:6789], mon addrv is [v2:172.21.15.33:3300,v1:172.21.15.33:6789] 2024-09-21T09:41:42.556 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.33 metric 100 2024-09-21T09:41:42.556 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.33 metric 100 2024-09-21T09:41:42.559 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-21T09:41:42.559 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-21T09:41:42.559 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1785sec hoplimit 64 pref medium 2024-09-21T09:41:42.562 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-21T09:41:42.562 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-21T09:41:42.562 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:41:42.562 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-21T09:41:42.562 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fea5:d294/64 scope link 2024-09-21T09:41:42.562 INFO:teuthology.orchestra.run.smithi033.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-21T09:41:42.563 INFO:teuthology.orchestra.run.smithi033.stdout:Mon IP `172.21.15.33` is in CIDR network `172.21.0.0/20` 2024-09-21T09:41:42.563 INFO:teuthology.orchestra.run.smithi033.stdout:Mon IP `172.21.15.33` is in CIDR network `172.21.0.0/20` 2024-09-21T09:41:42.563 INFO:teuthology.orchestra.run.smithi033.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-21T09:41:42.563 INFO:teuthology.orchestra.run.smithi033.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-21T09:41:42.564 INFO:teuthology.orchestra.run.smithi033.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:41:43.001 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stdout fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 2024-09-21T09:41:43.001 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-21T09:41:43.001 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Getting image source signatures 2024-09-21T09:41:43.002 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Copying blob sha256:3ba00a516579693832719224d909a7cd5333511394d4f35f95e899e50f0bfaeb 2024-09-21T09:41:43.002 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-21T09:41:43.002 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Copying config sha256:fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 2024-09-21T09:41:43.002 INFO:teuthology.orchestra.run.smithi033.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-21T09:41:43.981 INFO:teuthology.orchestra.run.smithi033.stdout:ceph: stdout ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:41:43.981 INFO:teuthology.orchestra.run.smithi033.stdout:Ceph version: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:41:43.981 INFO:teuthology.orchestra.run.smithi033.stdout:Extracting ceph user uid/gid from container image... 2024-09-21T09:41:44.955 INFO:teuthology.orchestra.run.smithi033.stdout:stat: stdout 167 167 2024-09-21T09:41:44.955 INFO:teuthology.orchestra.run.smithi033.stdout:Creating initial keys... 2024-09-21T09:41:45.960 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph-authtool: stdout AQDZlO5mcycgExAAujEp6JmdJLp8Bf1yFACLLA== 2024-09-21T09:41:47.033 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph-authtool: stdout AQDalO5mTlfdExAAg1nkVFtlF12HuPsWbhem3w== 2024-09-21T09:41:48.080 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph-authtool: stdout AQDblO5m/6JBFxAAqhBvjBRhhu3auLG9gADc2w== 2024-09-21T09:41:48.081 INFO:teuthology.orchestra.run.smithi033.stdout:Creating initial monmap... 2024-09-21T09:41:49.095 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:41:49.095 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-21T09:41:49.095 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:49.095 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:41:49.096 INFO:teuthology.orchestra.run.smithi033.stdout:monmaptool for a [v2:172.21.15.33:3300,v1:172.21.15.33:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-21T09:41:49.096 INFO:teuthology.orchestra.run.smithi033.stdout:setting min_mon_release = octopus 2024-09-21T09:41:49.096 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: set fsid to a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:49.096 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-21T09:41:49.096 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:41:49.096 INFO:teuthology.orchestra.run.smithi033.stdout:Creating mon... 2024-09-21T09:41:50.076 INFO:teuthology.orchestra.run.smithi033.stdout:create mon.a on 2024-09-21T09:41:50.281 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-21T09:41:50.434 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-21T09:41:50.590 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-a29e8460-77fd-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-a29e8460-77fd-11ef-baf6-efdc52797490.target. 2024-09-21T09:41:50.590 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-a29e8460-77fd-11ef-baf6-efdc52797490.target → /etc/systemd/system/ceph-a29e8460-77fd-11ef-baf6-efdc52797490.target. 2024-09-21T09:41:50.772 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a 2024-09-21T09:41:50.772 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Failed to reset failed state of unit ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a.service: Unit ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a.service not loaded. 2024-09-21T09:41:50.941 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-a29e8460-77fd-11ef-baf6-efdc52797490.target.wants/ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a.service → /etc/systemd/system/ceph-a29e8460-77fd-11ef-baf6-efdc52797490@.service. 2024-09-21T09:41:51.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:51 smithi033 podman[27458]: 2024-09-21 09:41:51.388253417 +0000 UTC m=+0.326036438 container start 3511a187baaf3ae1c3858dee47372d15783a42d01b86fbb74e2bf28c1de9072f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T09:41:51.590 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:41:51.591 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout disabled 2024-09-21T09:41:51.598 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:41:51.598 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-09-21T09:41:51.598 INFO:teuthology.orchestra.run.smithi033.stdout:firewalld.service is not enabled 2024-09-21T09:41:51.598 INFO:teuthology.orchestra.run.smithi033.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:41:51.598 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mon to start... 2024-09-21T09:41:51.598 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mon... 2024-09-21T09:41:51.703 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:51 smithi033 ceph-mon[27495]: mkfs a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:51.703 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:51 smithi033 ceph-mon[27495]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:41:51.703 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:51 smithi033 bash[27458]: 3511a187baaf3ae1c3858dee47372d15783a42d01b86fbb74e2bf28c1de9072f 2024-09-21T09:41:51.703 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:51 smithi033 systemd[1]: Started Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:41:52.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:52 smithi033 ceph-mon[27495]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:41:52.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:52 smithi033 ceph-mon[27495]: monmap e1: 1 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:41:52.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:52 smithi033 ceph-mon[27495]: fsmap 2024-09-21T09:41:52.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:52 smithi033 ceph-mon[27495]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:41:52.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:52 smithi033 ceph-mon[27495]: mgrmap e1: no daemons active 2024-09-21T09:41:52.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:52 smithi033 ceph-mon[27495]: from='client.? 172.21.15.33:0/2042666495' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-21T09:41:52.653 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout cluster: 2024-09-21T09:41:52.654 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout id: a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout services: 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.653159s) 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout data: 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout pgs: 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:mon is available 2024-09-21T09:41:52.655 INFO:teuthology.orchestra.run.smithi033.stdout:Assimilating anything we can from ceph.conf... 2024-09-21T09:41:53.594 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:41:53.594 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:41:53.594 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout fsid = a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:53.594 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-21T09:41:53.594 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.33:3300,v1:172.21.15.33:6789] 2024-09-21T09:41:53.594 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:41:53.594 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:41:53.594 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:41:53.595 INFO:teuthology.orchestra.run.smithi033.stdout:Generating new minimal ceph.conf... 2024-09-21T09:41:53.698 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:53 smithi033 ceph-mon[27495]: from='client.? 172.21.15.33:0/1718582170' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:41:53.699 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:53 smithi033 ceph-mon[27495]: from='client.? 172.21.15.33:0/1718582170' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-21T09:41:54.662 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:54 smithi033 ceph-mon[27495]: from='client.? 172.21.15.33:0/829065499' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:41:54.665 INFO:teuthology.orchestra.run.smithi033.stdout:Restarting the monitor... 2024-09-21T09:41:54.991 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:54 smithi033 systemd[1]: Stopping Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:54 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a[27472]: 2024-09-21T09:41:54.988+0000 7f6c64dd1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:54 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a[27472]: 2024-09-21T09:41:54.988+0000 7f6c64dd1640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-21T09:41:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:55 smithi033 podman[27815]: 2024-09-21 09:41:55.20478358 +0000 UTC m=+0.368419754 container died 3511a187baaf3ae1c3858dee47372d15783a42d01b86fbb74e2bf28c1de9072f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T09:41:55.716 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:55 smithi033 podman[27815]: 2024-09-21 09:41:55.514621964 +0000 UTC m=+0.678258145 container cleanup 3511a187baaf3ae1c3858dee47372d15783a42d01b86fbb74e2bf28c1de9072f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T09:41:55.716 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:55 smithi033 bash[27815]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:41:55.995 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:55 smithi033 podman[27832]: 2024-09-21 09:41:55.71523907 +0000 UTC m=+0.511542845 container remove 3511a187baaf3ae1c3858dee47372d15783a42d01b86fbb74e2bf28c1de9072f (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T09:41:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:55 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:41:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:55 smithi033 systemd[1]: Stopped Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:41:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:55 smithi033 systemd[1]: Starting Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:41:56.411 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 podman[27931]: 2024-09-21 09:41:56.115976613 +0000 UTC m=+0.096716719 container create 220cb0bc67bb9d33e368b771580239686f554a4c6452b3b2354cc0ef7d2134de (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T09:41:56.411 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 podman[27931]: 2024-09-21 09:41:56.035431761 +0000 UTC m=+0.016171887 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:41:56.411 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 podman[27931]: 2024-09-21 09:41:56.326962903 +0000 UTC m=+0.307703026 container init 220cb0bc67bb9d33e368b771580239686f554a4c6452b3b2354cc0ef7d2134de (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 podman[27931]: 2024-09-21 09:41:56.331572302 +0000 UTC m=+0.312312409 container start 220cb0bc67bb9d33e368b771580239686f554a4c6452b3b2354cc0ef7d2134de (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable), process ceph-mon, pid 7 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: pidfile_write: ignore empty --pid-file 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: load: jerasure load: lrc 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: RocksDB version: 6.15.5 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Compile date Sep 20 2024 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: DB SUMMARY 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: DB Session ID: 4S7DLI3O6EHDR8WUK7UP 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-21T09:41:56.412 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-21T09:41:56.413 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83686 ; 2024-09-21T09:41:56.413 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:41:56.413 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:41:56.413 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.env: 0x558552af5440 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.fs: Posix File System 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.info_log: 0x558554262ec0 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.statistics: (nil) 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.use_fsync: 0 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:41:56.414 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.db_log_dir: 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:41:56.415 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.write_buffer_manager: 0x5585543207e0 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.unordered_write: 0 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:41:56.416 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.row_cache: None 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.wal_filter: None 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.preserve_deletes: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:41:56.417 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:41:56.418 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_open_files: -1 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Compression algorithms supported: 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: kZSTD supported: 0 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:41:56.419 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.merge_operator: 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_filter: None 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558554209a38) 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:41:56.420 INFO:journalctl@ceph.mon.a.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_type: 0 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_block_index_type: 0 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_shortening: 1 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: hash_index_allow_collision: 1 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: checksum: 1 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: no_block_cache: 0 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache: 0x55855423f1f0 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_options: 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: capacity : 536870912 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: num_shard_bits : 4 2024-09-21T09:41:56.421 INFO:journalctl@ceph.mon.a.smithi033.stdout: strict_capacity_limit : 0 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_compressed: (nil) 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: persistent_cache: (nil) 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_size: 4096 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_size_deviation: 10 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_restart_interval: 16 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_block_restart_interval: 1 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: metadata_block_size: 4096 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: partition_filters: 0 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: use_delta_encoding: 1 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: whole_key_filtering: 1 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: verify_compression: 0 2024-09-21T09:41:56.422 INFO:journalctl@ceph.mon.a.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout: format_version: 4 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout: enable_index_compression: 1 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_align: 0 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compression: NoCompression 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.num_levels: 7 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:41:56.423 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:41:56.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:41:56.425 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:41:56.426 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:41:56.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:41:56.428 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:41:56.429 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:41:56.429 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.ttl: 2592000 2024-09-21T09:41:56.429 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:41:56.429 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:41:56.430 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:41:56.430 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:41:56.430 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911716355694, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911716356465, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80259, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78501, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10130, "raw_average_key_size": 47, "raw_value_size": 72714, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726911716, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "5d0733d4-c276-4395-be99-ec77a7e2a0d0", "db_session_id": "4S7DLI3O6EHDR8WUK7UP"}} 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911716356896, "job": 1, "event": "recovery_finished"} 2024-09-21T09:41:56.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x558554272700 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: DB pointer 0x558554242000 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** DB Stats ** 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:41:56.432 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.1 0.00 0.00 1 0.000 0 0 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.1 0.00 0.00 1 0.000 0 0 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.1 0.00 0.00 1 0.000 0 0 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 153.1 0.00 0.00 1 0.000 0 0 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:41:56.433 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative compaction: 0.00 GB write, 23.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval compaction: 0.00 GB write, 23.45 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: L0 2/0 80.16 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.1 0.00 0.00 1 0.000 0 0 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: Sum 2/0 80.16 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 153.1 0.00 0.00 1 0.000 0 0 2024-09-21T09:41:56.434 INFO:journalctl@ceph.mon.a.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 153.1 0.00 0.00 1 0.000 0 0 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative compaction: 0.00 GB write, 23.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:41:56.435 INFO:journalctl@ceph.mon.a.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: starting mon.a rank 0 at public addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] at bind addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a@-1(???) e1 preinit fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a@-1(???).mds e1 new map 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a@-1(???).mds e1 print_map 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout: e1 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout: legacy client fscid: -1 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout: No filesystems configured 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:41:56.436 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:41:56.437 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:41:56.437 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-21T09:41:56.437 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 bash[27931]: 220cb0bc67bb9d33e368b771580239686f554a4c6452b3b2354cc0ef7d2134de 2024-09-21T09:41:56.437 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 systemd[1]: Started Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:41:56.437 INFO:teuthology.orchestra.run.smithi033.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-21T09:41:56.745 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-21T09:41:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: monmap e1: 1 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:41:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: fsmap 2024-09-21T09:41:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: osdmap e1: 0 total, 0 up, 0 in 2024-09-21T09:41:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:56 smithi033 ceph-mon[27969]: mgrmap e1: no daemons active 2024-09-21T09:41:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:41:57 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3996410183' entity='client.admin' 2024-09-21T09:42:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-21T09:42:02.034 INFO:teuthology.orchestra.run.smithi033.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:42:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:Creating mgr... 2024-09-21T09:42:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-21T09:42:02.035 INFO:teuthology.orchestra.run.smithi033.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-21T09:42:02.300 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y 2024-09-21T09:42:02.301 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Failed to reset failed state of unit ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y.service: Unit ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y.service not loaded. 2024-09-21T09:42:02.660 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-a29e8460-77fd-11ef-baf6-efdc52797490.target.wants/ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y.service → /etc/systemd/system/ceph-a29e8460-77fd-11ef-baf6-efdc52797490@.service. 2024-09-21T09:42:02.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:02 smithi033 podman[28207]: 2024-09-21 09:42:02.794336309 +0000 UTC m=+0.016877643 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph:quincy 2024-09-21T09:42:03.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:03 smithi033 podman[28207]: 2024-09-21 09:42:03.729048766 +0000 UTC m=+0.951590101 container create 21061eedc1543bd6187bb850b6fee9b5edfd848ba05dbceec926b4325b838093 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, GIT_CLEAN=True) 2024-09-21T09:42:06.139 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:05 smithi033 podman[28207]: 2024-09-21 09:42:05.879800909 +0000 UTC m=+3.102342250 container init 21061eedc1543bd6187bb850b6fee9b5edfd848ba05dbceec926b4325b838093 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:42:06.139 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:05 smithi033 podman[28207]: 2024-09-21 09:42:05.887041618 +0000 UTC m=+3.109582941 container start 21061eedc1543bd6187bb850b6fee9b5edfd848ba05dbceec926b4325b838093 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T09:42:06.139 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:06.011+0000 7f1dbe12fec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:42:06.139 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:06.069+0000 7f1dbe12fec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:42:06.139 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:06.136+0000 7f1dbe12fec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:42:06.410 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:06.265+0000 7f1dbe12fec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:42:06.681 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:06.407+0000 7f1dbe12fec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:42:06.726 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:42:06.726 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout disabled 2024-09-21T09:42:06.734 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:42:06.734 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-09-21T09:42:06.734 INFO:teuthology.orchestra.run.smithi033.stdout:firewalld.service is not enabled 2024-09-21T09:42:06.734 INFO:teuthology.orchestra.run.smithi033.stdout:Not possible to enable service . firewalld.service is not available 2024-09-21T09:42:06.744 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:42:06.744 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout disabled 2024-09-21T09:42:06.753 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:42:06.753 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-09-21T09:42:06.753 INFO:teuthology.orchestra.run.smithi033.stdout:firewalld.service is not enabled 2024-09-21T09:42:06.753 INFO:teuthology.orchestra.run.smithi033.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-21T09:42:06.753 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mgr to start... 2024-09-21T09:42:06.753 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mgr... 2024-09-21T09:42:06.995 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:06.678+0000 7f1dbe12fec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:42:06.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 bash[28207]: 21061eedc1543bd6187bb850b6fee9b5edfd848ba05dbceec926b4325b838093 2024-09-21T09:42:06.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 systemd[1]: Started Ceph mgr.y for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:42:06.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:06.749+0000 7f1dbe12fec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:42:06.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:06.924+0000 7f1dbe12fec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:42:07.745 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:07 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/693964100' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:42:08.115 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:07.830+0000 7f1dbe12fec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:42:08.116 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:07.898+0000 7f1dbe12fec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:42:08.120 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:42:08.120 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:42:08.120 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsid": "a29e8460-77fd-11ef-baf6-efdc52797490", 2024-09-21T09:42:08.120 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:42:08.120 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:42:08.120 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:42:08.120 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_age": 11, 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:42:08.121 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:08.122 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:42:08.122 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:42:08.122 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:42:08.122 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:42:08.122 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:42:08.122 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:42:08.125 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:42:08.126 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:08.127 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:41:51.424676+0000", 2024-09-21T09:42:08.127 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:42:08.127 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:08.127 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:42:08.127 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:42:08.127 INFO:teuthology.orchestra.run.smithi033.stdout:mgr not available, waiting (1/15)... 2024-09-21T09:42:08.950 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:08.645+0000 7f1dbe12fec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:42:08.950 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:08.760+0000 7f1dbe12fec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:42:08.950 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:08.886+0000 7f1dbe12fec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:42:08.950 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:08.947+0000 7f1dbe12fec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:42:09.209 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.077+0000 7f1dbe12fec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:42:09.209 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.147+0000 7f1dbe12fec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:42:09.210 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.207+0000 7f1dbe12fec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:42:09.482 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.346+0000 7f1dbe12fec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:42:09.483 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.412+0000 7f1dbe12fec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:42:09.483 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.480+0000 7f1dbe12fec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:42:09.745 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.564+0000 7f1dbe12fec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:42:09.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.623+0000 7f1dbe12fec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:42:09.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:09.682+0000 7f1dbe12fec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:42:10.495 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:10.467+0000 7f1dbe12fec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:42:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: Activating manager daemon y 2024-09-21T09:42:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: mgrmap e2: y(active, starting, since 0.00293721s) 2024-09-21T09:42:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:42:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:42:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:42:10.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:42:10.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:42:10.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: Manager daemon y is now available 2024-09-21T09:42:10.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:42:10.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:42:10.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' 2024-09-21T09:42:10.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' 2024-09-21T09:42:10.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:10 smithi033 ceph-mon[27969]: from='mgr.14100 172.21.15.33:0/480314470' entity='mgr.y' 2024-09-21T09:42:11.268 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsid": "a29e8460-77fd-11ef-baf6-efdc52797490", 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:42:11.269 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:42:11.270 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:42:11.273 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:42:11.273 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:42:11.273 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:42:11.274 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:11.275 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:41:51.424676+0000", 2024-09-21T09:42:11.276 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:42:11.276 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:11.276 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:42:11.276 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:42:11.276 INFO:teuthology.orchestra.run.smithi033.stdout:mgr not available, waiting (2/15)... 2024-09-21T09:42:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:11 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3743091207' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:42:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:11 smithi033 ceph-mon[27969]: mgrmap e3: y(active, since 1.00706s) 2024-09-21T09:42:13.714 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:13 smithi033 ceph-mon[27969]: mgrmap e4: y(active, since 2s) 2024-09-21T09:42:14.551 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:14 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/46901812' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-21T09:42:14.559 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:42:14.559 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:42:14.559 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsid": "a29e8460-77fd-11ef-baf6-efdc52797490", 2024-09-21T09:42:14.559 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "health": { 2024-09-21T09:42:14.559 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-21T09:42:14.559 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 0 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "a" 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "quorum_age": 17, 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-21T09:42:14.560 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-21T09:42:14.561 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-21T09:42:14.562 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "restful" 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ], 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "modified": "2024-09-21T09:41:51.424676+0000", 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout }, 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:42:14.563 INFO:teuthology.orchestra.run.smithi033.stdout:mgr is available 2024-09-21T09:42:15.679 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:42:15.679 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [global] 2024-09-21T09:42:15.679 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout fsid = a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:42:15.679 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-21T09:42:15.679 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-21T09:42:15.679 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-21T09:42:15.679 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout [osd] 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-21T09:42:15.680 INFO:teuthology.orchestra.run.smithi033.stdout:Enabling cephadm module... 2024-09-21T09:42:15.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:15 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3627480680' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-21T09:42:16.796 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:16 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3790780485' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-21T09:42:16.796 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ignoring --setuser ceph since I am not root 2024-09-21T09:42:16.797 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ignoring --setgroup ceph since I am not root 2024-09-21T09:42:16.797 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:16.668+0000 7f6e5f0b1ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:42:16.797 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:16.725+0000 7f6e5f0b1ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:42:16.797 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:16.793+0000 7f6e5f0b1ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:42:17.067 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:16.923+0000 7f6e5f0b1ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:42:17.067 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:17.065+0000 7f6e5f0b1ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:42:17.618 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:17 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3790780485' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-21T09:42:17.618 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:17 smithi033 ceph-mon[27969]: mgrmap e5: y(active, since 6s) 2024-09-21T09:42:17.618 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:17.344+0000 7f6e5f0b1ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:42:17.619 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:17.412+0000 7f6e5f0b1ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:42:17.619 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:17.582+0000 7f6e5f0b1ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:42:18.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:18.467+0000 7f6e5f0b1ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:42:18.731 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:18.534+0000 7f6e5f0b1ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:42:18.731 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:18 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3785623437' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:42:18.734 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:42:18.734 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-21T09:42:18.734 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:42:18.734 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:42:18.734 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:42:18.734 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:42:18.734 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for the mgr to restart... 2024-09-21T09:42:18.734 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mgr epoch 5... 2024-09-21T09:42:19.592 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:19 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:19.288+0000 7f6e5f0b1ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:42:19.592 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:19 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:19.403+0000 7f6e5f0b1ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:42:19.592 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:19 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:19.528+0000 7f6e5f0b1ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:42:19.592 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:19 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:19.589+0000 7f6e5f0b1ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:42:19.850 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:19 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:19.719+0000 7f6e5f0b1ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:42:19.850 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:19 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:19.788+0000 7f6e5f0b1ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:42:19.850 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:19 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:19.848+0000 7f6e5f0b1ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:42:20.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:19 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:19.986+0000 7f6e5f0b1ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:42:20.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:20.052+0000 7f6e5f0b1ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:42:20.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:20.119+0000 7f6e5f0b1ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:42:20.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:20.203+0000 7f6e5f0b1ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:42:20.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:20.262+0000 7f6e5f0b1ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:42:20.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:20.321+0000 7f6e5f0b1ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:42:21.495 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:21.095+0000 7f6e5f0b1ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: Active manager daemon y restarted 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: Activating manager daemon y 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: osdmap e2: 0 total, 0 up, 0 in 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: mgrmap e6: y(active, starting, since 0.00287595s) 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: Manager daemon y is now available 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:21.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:21.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:21.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:21.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:42:21.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:21 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:42:22.403 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:22 smithi033 ceph-mon[27969]: Found migration_current of "None". Setting to last migration. 2024-09-21T09:42:22.403 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:22 smithi033 ceph-mon[27969]: mgrmap e7: y(active, since 1.00544s) 2024-09-21T09:42:22.665 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:42:22.666 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-21T09:42:22.666 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:42:22.666 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:42:22.666 INFO:teuthology.orchestra.run.smithi033.stdout:mgr epoch 5 is available 2024-09-21T09:42:22.666 INFO:teuthology.orchestra.run.smithi033.stdout:Setting orchestrator backend to cephadm... 2024-09-21T09:42:23.298 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-mon[27969]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:42:23.298 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-mon[27969]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:42:23.853 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:23.677+0000 7f6e11460640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:42:23] ENGINE Error in HTTPServer.serve 2024-09-21T09:42:23.853 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: Traceback (most recent call last): 2024-09-21T09:42:23.853 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:42:23.853 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._connections.run(self.expiration_interval) 2024-09-21T09:42:23.853 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:42:23.853 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._run(expiration_interval) 2024-09-21T09:42:23.853 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: s = self.context.wrap_socket( 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: return self.sslsocket_class._create( 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self.do_handshake() 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._sslobj.do_handshake() 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:42:23.854 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:24.458 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:24 smithi033 ceph-mon[27969]: mgrmap e8: y(active, since 2s) 2024-09-21T09:42:24.458 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:24 smithi033 ceph-mon[27969]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:24 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:24 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:24 smithi033 ceph-mon[27969]: [21/Sep/2024:09:42:23] ENGINE Bus STARTING 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:24 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:24.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:24 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:25.003 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-21T09:42:25.004 INFO:teuthology.orchestra.run.smithi033.stdout:Generating ssh key... 2024-09-21T09:42:25.445 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-mon[27969]: [21/Sep/2024:09:42:23] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:42:25.445 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-mon[27969]: [21/Sep/2024:09:42:23] ENGINE Bus STARTED 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-mon[27969]: [21/Sep/2024:09:42:23] ENGINE Error in HTTPServer.serve 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:42:25.446 INFO:journalctl@ceph.mon.a.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:42:25.447 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:42:25.447 INFO:journalctl@ceph.mon.a.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:42:25.447 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:42:25.447 INFO:journalctl@ceph.mon.a.smithi033.stdout: self.do_handshake() 2024-09-21T09:42:25.447 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:42:25.447 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:42:25.447 INFO:journalctl@ceph.mon.a.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:42:25.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-mon[27969]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:26.199 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: Generating public/private rsa key pair. 2024-09-21T09:42:26.199 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: Your identification has been saved in /tmp/tmpiki44b3j/key 2024-09-21T09:42:26.199 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: Your public key has been saved in /tmp/tmpiki44b3j/key.pub 2024-09-21T09:42:26.199 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: The key fingerprint is: 2024-09-21T09:42:26.199 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: SHA256:qj3jkcGyuj3pBueCqMcBry+3ZTL1BGFUJd/ek5zvSMk ceph-a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:42:26.199 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: The key's randomart image is: 2024-09-21T09:42:26.199 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: +---[RSA 3072]----+ 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: | .+.o.. | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: | . . o . | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: | . . . | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: |. o . o o | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: | o o + S . * | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: | +..= + . + | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: |.+o=+.= E . | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: |= =B=oo. . o | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: |o==*+ooo . . | 2024-09-21T09:42:26.200 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: +----[SHA256]-----+ 2024-09-21T09:42:27.082 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:26 smithi033 ceph-mon[27969]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:27.082 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:26 smithi033 ceph-mon[27969]: Generating ssh key... 2024-09-21T09:42:27.082 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:26 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:27.083 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:26 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:27.675 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4IJbig9L7QonPzn4MwnH4u8of8NZZV1+26t6lRf1PIc7wEi+au8EYtQDvyRgVM+RLpKlm3nFJMoo/eWmaILQ2Nv1x95czPJ375L5dlCnFjge8W7qoQ29toJw+0iPdvpixvlMAnMkES8T5zPEhrxOeDHA6uIK55kCTa6ryEpxp+4amVG1grKfPX3Ljo3m5otCYF8pUooIZ6jZ4xSLyQNKdwR7XrW1KrShOIMx6Jc15liM4kDaCX53UboP+aGldpkWk3OP+gJazLVsIjFy5YLaoPBtClgVqcT7Wi1MjtLtgdVl9rLmdFOJhySze680DF8WrUVS6TpoGJj4JBNuYtvJkZe4HF0XffuDI9rHErGzb8lJv5Pim0sOnAtMHJS/eVyZp7nm0VMx4pteTD8WXowSV6j7bgQ4KfX+GJ8HXUzXnf1hS8dJylTCSarVmFsQhBKqx+js4yqxStNYNYV0cxjCFWhEnkk3TV9hfBDq0k1g7L7J2a4dFs9QPpPH9qfCkrdk= ceph-a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:42:27.676 INFO:teuthology.orchestra.run.smithi033.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-21T09:42:27.676 INFO:teuthology.orchestra.run.smithi033.stdout:Adding key to root@localhost authorized_keys... 2024-09-21T09:42:27.680 INFO:teuthology.orchestra.run.smithi033.stdout:Adding host smithi033... 2024-09-21T09:42:27.935 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:27 smithi033 ceph-mon[27969]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:28 smithi033 ceph-mon[27969]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi033", "addr": "172.21.15.33", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:29.950 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:29 smithi033 ceph-mon[27969]: Deploying cephadm binary to smithi033 2024-09-21T09:42:32.267 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout Added host 'smithi033' with addr '172.21.15.33' 2024-09-21T09:42:32.267 INFO:teuthology.orchestra.run.smithi033.stdout:Deploying unmanaged mon service... 2024-09-21T09:42:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:32 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:32 smithi033 ceph-mon[27969]: Added host smithi033 2024-09-21T09:42:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:32 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:33.772 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-21T09:42:33.772 INFO:teuthology.orchestra.run.smithi033.stdout:Deploying unmanaged mgr service... 2024-09-21T09:42:34.239 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:33 smithi033 ceph-mon[27969]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:34.239 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:33 smithi033 ceph-mon[27969]: Saving service mon spec with placement count:5 2024-09-21T09:42:34.239 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:33 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:35.838 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:35 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:35.839 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:35 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:35.839 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:35 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:36.117 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-21T09:42:36.745 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:36 smithi033 ceph-mon[27969]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:36.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:36 smithi033 ceph-mon[27969]: Saving service mgr spec with placement count:2 2024-09-21T09:42:37.992 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:37 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:37.992 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:37 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/167503840' entity='client.admin' 2024-09-21T09:42:39.241 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:38 smithi033 ceph-mon[27969]: from='mgr.14118 172.21.15.33:0/1271686071' entity='mgr.y' 2024-09-21T09:42:39.241 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:38 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1167652953' entity='client.admin' 2024-09-21T09:42:39.503 INFO:teuthology.orchestra.run.smithi033.stdout:Enabling the dashboard module... 2024-09-21T09:42:40.916 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:40 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1357312746' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-21T09:42:40.916 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:40 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ignoring --setuser ceph since I am not root 2024-09-21T09:42:40.916 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:40 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ignoring --setgroup ceph since I am not root 2024-09-21T09:42:40.917 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:40 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:40.788+0000 7fb20debbec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:42:40.917 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:40 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:40.846+0000 7fb20debbec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:42:40.917 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:40 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:40.915+0000 7fb20debbec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:42:41.193 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:41.048+0000 7fb20debbec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:42:41.193 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:41.191+0000 7fb20debbec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:42:41.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:41 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1357312746' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-21T09:42:41.722 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:41 smithi033 ceph-mon[27969]: mgrmap e9: y(active, since 19s) 2024-09-21T09:42:41.723 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:41.466+0000 7fb20debbec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:42:41.723 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:41.529+0000 7fb20debbec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:42:41.995 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:41.718+0000 7fb20debbec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:42:42.639 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:42:42.639 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-21T09:42:42.639 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-21T09:42:42.639 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-21T09:42:42.639 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-21T09:42:42.639 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:42:42.640 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for the mgr to restart... 2024-09-21T09:42:42.640 INFO:teuthology.orchestra.run.smithi033.stdout:Waiting for mgr epoch 9... 2024-09-21T09:42:42.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:42 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/4280832593' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-21T09:42:42.904 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:42 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:42.649+0000 7fb20debbec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:42:42.904 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:42 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:42.722+0000 7fb20debbec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:42:43.745 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:43.485+0000 7fb20debbec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:42:43.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:43.602+0000 7fb20debbec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:42:43.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:43.729+0000 7fb20debbec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:42:44.056 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:43.791+0000 7fb20debbec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:42:44.056 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:43.923+0000 7fb20debbec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:42:44.057 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:43.993+0000 7fb20debbec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:42:44.057 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:44.054+0000 7fb20debbec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:42:44.332 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:44.195+0000 7fb20debbec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:42:44.332 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:44.262+0000 7fb20debbec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:42:44.333 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:44.330+0000 7fb20debbec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:42:44.745 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:44.415+0000 7fb20debbec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:42:44.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:44.475+0000 7fb20debbec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:42:44.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:44.535+0000 7fb20debbec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:42:45.611 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:45.321+0000 7fb20debbec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: Active manager daemon y restarted 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: Activating manager daemon y 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: osdmap e3: 0 total, 0 up, 0 in 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: mgrmap e10: y(active, starting, since 0.00313847s) 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: Manager daemon y is now available 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:45.612 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:42:45.613 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:42:47.273 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout { 2024-09-21T09:42:47.273 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-21T09:42:47.273 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-21T09:42:47.273 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout } 2024-09-21T09:42:47.273 INFO:teuthology.orchestra.run.smithi033.stdout:mgr epoch 9 is available 2024-09-21T09:42:47.273 INFO:teuthology.orchestra.run.smithi033.stdout:Generating a dashboard self-signed certificate... 2024-09-21T09:42:47.531 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-mon[27969]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-21T09:42:47.531 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-mon[27969]: mgrmap e11: y(active, since 1.00568s) 2024-09-21T09:42:47.531 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-mon[27969]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-21T09:42:47.531 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:47.531 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:47.531 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:47.285+0000 7fb1bea19640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:42:47] ENGINE Error in HTTPServer.serve 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: Traceback (most recent call last): 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._connections.run(self.expiration_interval) 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._run(expiration_interval) 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: s = self.context.wrap_socket( 2024-09-21T09:42:47.532 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:42:47.533 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: return self.sslsocket_class._create( 2024-09-21T09:42:47.533 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:42:47.533 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self.do_handshake() 2024-09-21T09:42:47.533 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:42:47.533 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._sslobj.do_handshake() 2024-09-21T09:42:47.533 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:42:47.533 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:42:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:42:48.538 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-21T09:42:48.538 INFO:teuthology.orchestra.run.smithi033.stdout:Creating initial admin user... 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:48 smithi033 ceph-mon[27969]: [21/Sep/2024:09:42:47] ENGINE Bus STARTING 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:48 smithi033 ceph-mon[27969]: [21/Sep/2024:09:42:47] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:48 smithi033 ceph-mon[27969]: [21/Sep/2024:09:42:47] ENGINE Bus STARTED 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:48 smithi033 ceph-mon[27969]: [21/Sep/2024:09:42:47] ENGINE Error in HTTPServer.serve 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:42:48.671 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: self.do_handshake() 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:48 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:48 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:48.672 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:48 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:48.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:48 smithi033 ceph-mon[27969]: mgrmap e12: y(active, since 2s) 2024-09-21T09:42:49.449 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:49 smithi033 ceph-mon[27969]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:50.248 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$nueTuwRpOh6iuPctpSiQLe88Vv7y2SzH2SJbXMb2ZXZ3NYOKtyL3m", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726911769, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-21T09:42:50.248 INFO:teuthology.orchestra.run.smithi033.stdout:Fetching dashboard port number... 2024-09-21T09:42:50.631 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:50 smithi033 ceph-mon[27969]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:42:50.631 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:50 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:51.637 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:51 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/93845516' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-21T09:42:51.890 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stdout 8443 2024-09-21T09:42:51.900 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-21T09:42:51.900 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout disabled 2024-09-21T09:42:51.908 INFO:teuthology.orchestra.run.smithi033.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-21T09:42:51.908 INFO:teuthology.orchestra.run.smithi033.stdout:systemctl: stdout inactive 2024-09-21T09:42:51.908 INFO:teuthology.orchestra.run.smithi033.stdout:firewalld.service is not enabled 2024-09-21T09:42:51.909 INFO:teuthology.orchestra.run.smithi033.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-21T09:42:51.911 INFO:teuthology.orchestra.run.smithi033.stdout:Ceph Dashboard is now available at: 2024-09-21T09:42:51.911 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:51.911 INFO:teuthology.orchestra.run.smithi033.stdout: URL: https://smithi033.front.sepia.ceph.com:8443/ 2024-09-21T09:42:51.911 INFO:teuthology.orchestra.run.smithi033.stdout: User: admin 2024-09-21T09:42:51.911 INFO:teuthology.orchestra.run.smithi033.stdout: Password: bf2aui6knq 2024-09-21T09:42:51.911 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:51.912 INFO:teuthology.orchestra.run.smithi033.stdout:Saving cluster configuration to /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config directory 2024-09-21T09:42:51.912 INFO:teuthology.orchestra.run.smithi033.stdout:Enabling autotune for osd_memory_target 2024-09-21T09:42:52.539 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:52 smithi033 ceph-mon[27969]: mgrmap e13: y(active, since 6s) 2024-09-21T09:42:54.386 INFO:teuthology.orchestra.run.smithi033.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-21T09:42:54.386 INFO:teuthology.orchestra.run.smithi033.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout:Or, if you are only running a single cluster on this host: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: ceph telemetry on 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout:For more information see: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:42:54.387 INFO:teuthology.orchestra.run.smithi033.stdout:Bootstrap complete. 2024-09-21T09:42:54.430 INFO:tasks.cephadm:Fetching config... 2024-09-21T09:42:54.431 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:42:54.431 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-21T09:42:54.448 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-21T09:42:54.449 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:42:54.449 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-21T09:42:54.506 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-21T09:42:54.506 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:42:54.506 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/keyring of=/dev/stdout 2024-09-21T09:42:54.573 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-21T09:42:54.574 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:42:54.574 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-21T09:42:54.630 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-21T09:42:54.631 DEBUG:teuthology.orchestra.run.smithi033:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4IJbig9L7QonPzn4MwnH4u8of8NZZV1+26t6lRf1PIc7wEi+au8EYtQDvyRgVM+RLpKlm3nFJMoo/eWmaILQ2Nv1x95czPJ375L5dlCnFjge8W7qoQ29toJw+0iPdvpixvlMAnMkES8T5zPEhrxOeDHA6uIK55kCTa6ryEpxp+4amVG1grKfPX3Ljo3m5otCYF8pUooIZ6jZ4xSLyQNKdwR7XrW1KrShOIMx6Jc15liM4kDaCX53UboP+aGldpkWk3OP+gJazLVsIjFy5YLaoPBtClgVqcT7Wi1MjtLtgdVl9rLmdFOJhySze680DF8WrUVS6TpoGJj4JBNuYtvJkZe4HF0XffuDI9rHErGzb8lJv5Pim0sOnAtMHJS/eVyZp7nm0VMx4pteTD8WXowSV6j7bgQ4KfX+GJ8HXUzXnf1hS8dJylTCSarVmFsQhBKqx+js4yqxStNYNYV0cxjCFWhEnkk3TV9hfBDq0k1g7L7J2a4dFs9QPpPH9qfCkrdk= ceph-a29e8460-77fd-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:42:54.712 INFO:teuthology.orchestra.run.smithi033.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4IJbig9L7QonPzn4MwnH4u8of8NZZV1+26t6lRf1PIc7wEi+au8EYtQDvyRgVM+RLpKlm3nFJMoo/eWmaILQ2Nv1x95czPJ375L5dlCnFjge8W7qoQ29toJw+0iPdvpixvlMAnMkES8T5zPEhrxOeDHA6uIK55kCTa6ryEpxp+4amVG1grKfPX3Ljo3m5otCYF8pUooIZ6jZ4xSLyQNKdwR7XrW1KrShOIMx6Jc15liM4kDaCX53UboP+aGldpkWk3OP+gJazLVsIjFy5YLaoPBtClgVqcT7Wi1MjtLtgdVl9rLmdFOJhySze680DF8WrUVS6TpoGJj4JBNuYtvJkZe4HF0XffuDI9rHErGzb8lJv5Pim0sOnAtMHJS/eVyZp7nm0VMx4pteTD8WXowSV6j7bgQ4KfX+GJ8HXUzXnf1hS8dJylTCSarVmFsQhBKqx+js4yqxStNYNYV0cxjCFWhEnkk3TV9hfBDq0k1g7L7J2a4dFs9QPpPH9qfCkrdk= ceph-a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:42:54.725 DEBUG:teuthology.orchestra.run.smithi105:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4IJbig9L7QonPzn4MwnH4u8of8NZZV1+26t6lRf1PIc7wEi+au8EYtQDvyRgVM+RLpKlm3nFJMoo/eWmaILQ2Nv1x95czPJ375L5dlCnFjge8W7qoQ29toJw+0iPdvpixvlMAnMkES8T5zPEhrxOeDHA6uIK55kCTa6ryEpxp+4amVG1grKfPX3Ljo3m5otCYF8pUooIZ6jZ4xSLyQNKdwR7XrW1KrShOIMx6Jc15liM4kDaCX53UboP+aGldpkWk3OP+gJazLVsIjFy5YLaoPBtClgVqcT7Wi1MjtLtgdVl9rLmdFOJhySze680DF8WrUVS6TpoGJj4JBNuYtvJkZe4HF0XffuDI9rHErGzb8lJv5Pim0sOnAtMHJS/eVyZp7nm0VMx4pteTD8WXowSV6j7bgQ4KfX+GJ8HXUzXnf1hS8dJylTCSarVmFsQhBKqx+js4yqxStNYNYV0cxjCFWhEnkk3TV9hfBDq0k1g7L7J2a4dFs9QPpPH9qfCkrdk= ceph-a29e8460-77fd-11ef-baf6-efdc52797490' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-21T09:42:54.773 INFO:teuthology.orchestra.run.smithi105.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC4IJbig9L7QonPzn4MwnH4u8of8NZZV1+26t6lRf1PIc7wEi+au8EYtQDvyRgVM+RLpKlm3nFJMoo/eWmaILQ2Nv1x95czPJ375L5dlCnFjge8W7qoQ29toJw+0iPdvpixvlMAnMkES8T5zPEhrxOeDHA6uIK55kCTa6ryEpxp+4amVG1grKfPX3Ljo3m5otCYF8pUooIZ6jZ4xSLyQNKdwR7XrW1KrShOIMx6Jc15liM4kDaCX53UboP+aGldpkWk3OP+gJazLVsIjFy5YLaoPBtClgVqcT7Wi1MjtLtgdVl9rLmdFOJhySze680DF8WrUVS6TpoGJj4JBNuYtvJkZe4HF0XffuDI9rHErGzb8lJv5Pim0sOnAtMHJS/eVyZp7nm0VMx4pteTD8WXowSV6j7bgQ4KfX+GJ8HXUzXnf1hS8dJylTCSarVmFsQhBKqx+js4yqxStNYNYV0cxjCFWhEnkk3TV9hfBDq0k1g7L7J2a4dFs9QPpPH9qfCkrdk= ceph-a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:42:54.787 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-21T09:42:54.995 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:54 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2254393829' entity='client.admin' 2024-09-21T09:42:55.045 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:42:57.745 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi033", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:42:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/4100364426' entity='client.admin' 2024-09-21T09:42:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:42:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:42:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:42:58.073 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-21T09:42:58.073 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-21T09:42:58.753 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:01.350 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi105 2024-09-21T09:43:01.350 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:43:01.350 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/etc/ceph/ceph.conf 2024-09-21T09:43:01.370 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:43:01.370 DEBUG:teuthology.orchestra.run.smithi105:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:43:01.429 INFO:tasks.cephadm:Adding host smithi105 to orchestrator... 2024-09-21T09:43:01.429 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch host add smithi105 2024-09-21T09:43:01.678 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:43:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:01.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:01.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:01.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:02.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:02 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:43:02.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:02 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:43:04.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:03 smithi033 ceph-mon[27969]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi105", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:05.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:04 smithi033 ceph-mon[27969]: Deploying cephadm binary to smithi105 2024-09-21T09:43:06.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:06 smithi033 ceph-mon[27969]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:06.875 INFO:teuthology.orchestra.run.smithi033.stdout:Added host 'smithi105' with addr '172.21.15.105' 2024-09-21T09:43:07.497 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch host ls --format=json 2024-09-21T09:43:07.748 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:08.123 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:08.123 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:07 smithi033 ceph-mon[27969]: Added host smithi105 2024-09-21T09:43:08.123 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:08.123 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:08.123 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:07 smithi033 ceph-mon[27969]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:08.123 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:09.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:09.529 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:43:09.529 INFO:teuthology.orchestra.run.smithi033.stdout:[{"addr": "172.21.15.33", "hostname": "smithi033", "labels": [], "status": ""}, {"addr": "172.21.15.105", "hostname": "smithi105", "labels": [], "status": ""}] 2024-09-21T09:43:10.115 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-21T09:43:10.116 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd crush tunables default 2024-09-21T09:43:10.363 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:10.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:10 smithi033 ceph-mon[27969]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:10.485 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:10 smithi033 ceph-mon[27969]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:43:12.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:12.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:12.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:12.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:12.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi105", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:43:12.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:12.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:43:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:43:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:43:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:13.012 INFO:teuthology.orchestra.run.smithi033.stderr:adjusted tunables profile to default 2024-09-21T09:43:13.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:13 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1166781143' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:43:13.648 INFO:tasks.cephadm:Adding mon.a on smithi033 2024-09-21T09:43:13.648 INFO:tasks.cephadm:Adding mon.c on smithi033 2024-09-21T09:43:13.648 INFO:tasks.cephadm:Adding mon.b on smithi105 2024-09-21T09:43:13.648 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch apply mon '3;smithi033:172.21.15.33=a;smithi033:[v2:172.21.15.33:3301,v1:172.21.15.33:6790]=c;smithi105:172.21.15.105=b' 2024-09-21T09:43:13.890 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:14.495 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:14 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1166781143' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:43:14.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:14 smithi033 ceph-mon[27969]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:43:14.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:14 smithi033 ceph-mon[27969]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:15.694 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled mon update... 2024-09-21T09:43:16.297 DEBUG:teuthology.orchestra.run.smithi033:mon.c> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.c.service 2024-09-21T09:43:16.300 DEBUG:teuthology.orchestra.run.smithi105:mon.b> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.b.service 2024-09-21T09:43:16.302 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:43:16.302 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:43:16.988 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi033:172.21.15.33=a;smithi033:[v2:172.21.15.33:3301,v1:172.21.15.33:6790]=c;smithi105:172.21.15.105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: Saving service mon spec with placement smithi033:172.21.15.33=a;smithi033:[v2:172.21.15.33:3301,v1:172.21.15.33:6790]=c;smithi105:172.21.15.105=b;count:3 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:16 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:17.995 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:17 smithi033 ceph-mon[27969]: Deploying daemon mon.b on smithi105 2024-09-21T09:43:18.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:18 smithi033 ceph-mon[27969]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:19.994 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:43:19.994 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":1,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","modified":"2024-09-21T09:41:48.442447Z","created":"2024-09-21T09:41:48.442447Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-21T09:43:19.994 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 1 2024-09-21T09:43:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:20 smithi033 ceph-mon[27969]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:20 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/3692585780' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:43:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:21 smithi105 ceph-mon[31286]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:43:21.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:21 smithi105 bash[31247]: c2741af3f853eba26e6ce9d6bec37c7f672802e9f6e40cb60e5f7a0f3cad5a74 2024-09-21T09:43:21.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:21 smithi105 systemd[1]: Started Ceph mon.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:43:21.584 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:43:21.585 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:43:21.833 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: mon.a calling monitor election 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: mon.b calling monitor election 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:26.473 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: monmap e2: 2 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: fsmap 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: mgrmap e13: y(active, since 40s) 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: overall HEALTH_OK 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:26.474 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: mon.a calling monitor election 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: mon.b calling monitor election 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:26.516 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: monmap e2: 2 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: fsmap 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: mgrmap e13: y(active, since 40s) 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: overall HEALTH_OK 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:26.517 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:26.598 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:43:26.598 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":2,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","modified":"2024-09-21T09:43:21.005284Z","created":"2024-09-21T09:41:48.442447Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-21T09:43:26.598 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 2 2024-09-21T09:43:27.156 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:27 smithi105 ceph-mon[31286]: Deploying daemon mon.c on smithi033 2024-09-21T09:43:27.156 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:27 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/3314274777' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:43:27.156 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:27 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:27.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:27 smithi033 ceph-mon[27969]: Deploying daemon mon.c on smithi033 2024-09-21T09:43:27.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:27 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/3314274777' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:43:27.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:27 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:28.180 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-21T09:43:28.181 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph mon dump -f json 2024-09-21T09:43:28.365 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:28 smithi033 ceph-mon[27969]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:28.423 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:43:28.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:28 smithi105 ceph-mon[31286]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.345 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 podman[39008]: 2024-09-21 09:43:29.135581748 +0000 UTC m=+0.016286789 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T09:43:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 podman[39008]: 2024-09-21 09:43:29.45189483 +0000 UTC m=+0.332599881 container init 0f8b68f40cc5bb185d5cf2e97bed9b851a36e8ab5b6f4f952bc425bcb70e97f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T09:43:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 podman[39008]: 2024-09-21 09:43:29.456609745 +0000 UTC m=+0.337314792 container start 0f8b68f40cc5bb185d5cf2e97bed9b851a36e8ab5b6f4f952bc425bcb70e97f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:43:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:43:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable), process ceph-mon, pid 7 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pidfile_write: ignore empty --pid-file 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: load: jerasure load: lrc 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: RocksDB version: 6.15.5 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Compile date Sep 20 2024 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: DB SUMMARY 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: DB Session ID: 3N94G7VXS9AE6RH4X9UR 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:43:29.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.env: 0x55680ee7c440 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.fs: Posix File System 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.info_log: 0x556810820ec0 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.statistics: (nil) 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.use_fsync: 0 2024-09-21T09:43:29.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:43:29.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:43:29.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.db_log_dir: 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:43:29.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.write_buffer_manager: 0x5568108de7b0 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:43:29.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.unordered_write: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.row_cache: None 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.wal_filter: None 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.preserve_deletes: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:43:29.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:43:29.753 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_open_files: -1 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Compression algorithms supported: 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: kZSTD supported: 0 2024-09-21T09:43:29.754 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.merge_operator: 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_filter: None 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:43:29.755 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5568107c7a38) 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_type: 0 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: data_block_index_type: 0 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_shortening: 1 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: hash_index_allow_collision: 1 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: checksum: 1 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: no_block_cache: 0 2024-09-21T09:43:29.756 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache: 0x5568107fd1f0 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_options: 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: capacity : 536870912 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: num_shard_bits : 4 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: strict_capacity_limit : 0 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_compressed: (nil) 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: persistent_cache: (nil) 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_size: 4096 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_size_deviation: 10 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_restart_interval: 16 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_block_restart_interval: 1 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: metadata_block_size: 4096 2024-09-21T09:43:29.757 INFO:journalctl@ceph.mon.c.smithi033.stdout: partition_filters: 0 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout: use_delta_encoding: 1 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout: whole_key_filtering: 1 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout: verify_compression: 0 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout: format_version: 4 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout: enable_index_compression: 1 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_align: 0 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compression: NoCompression 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:43:29.758 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.num_levels: 7 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:43:29.759 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:43:29.760 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.arena_block_size: 4194304 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:43:29.761 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:43:29.762 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.ttl: 2592000 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911809480990, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-21T09:43:29.763 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911809481297, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1696, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726911809, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "8c4b7523-fe02-42b8-a674-b369bcc4a640", "db_session_id": "3N94G7VXS9AE6RH4X9UR"}} 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726911809481651, "job": 1, "event": "recovery_finished"} 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x556810830700 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: DB pointer 0x556810800000 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** DB Stats ** 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:43:29.764 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:43:29.765 INFO:journalctl@ceph.mon.c.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:43:29.766 INFO:journalctl@ceph.mon.c.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: L0 1/0 1.66 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: Sum 1/0 1.66 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.6 0.00 0.00 1 0.000 0 0 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:43:29.767 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: using public_addrv [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: starting mon.c rank -1 at public addrs [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] at bind addrs [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(???) e0 preinit fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).mds e1 new map 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: e1 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:43:29.768 INFO:journalctl@ceph.mon.c.smithi033.stdout: legacy client fscid: -1 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout: No filesystems configured 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.769 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi105", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.770 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1166781143' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1166781143' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi033:172.21.15.33=a;smithi033:[v2:172.21.15.33:3301,v1:172.21.15.33:6790]=c;smithi105:172.21.15.105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: Saving service mon spec with placement smithi033:172.21.15.33=a;smithi033:[v2:172.21.15.33:3301,v1:172.21.15.33:6790]=c;smithi105:172.21.15.105=b;count:3 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:29.771 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: Deploying daemon mon.b on smithi105 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/3692585780' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.a calling monitor election 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.b calling monitor election 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.772 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: monmap e2: 2 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: fsmap 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mgrmap e13: y(active, since 40s) 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: overall HEALTH_OK 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: Deploying daemon mon.c on smithi033 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/3314274777' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:43:29.773 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:29.774 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:29.774 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 ceph-mon[39045]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-21T09:43:29.774 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 bash[39008]: 0f8b68f40cc5bb185d5cf2e97bed9b851a36e8ab5b6f4f952bc425bcb70e97f2 2024-09-21T09:43:29.774 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:29 smithi033 systemd[1]: Started Ceph mon.c for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: mon.a calling monitor election 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: mon.b calling monitor election 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: mon.c calling monitor election 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: fsmap 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: mgrmap e13: y(active, since 49s) 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: overall HEALTH_OK 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: mon.a calling monitor election 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: mon.b calling monitor election 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:34.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: mon.c calling monitor election 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: fsmap 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: mgrmap e13: y(active, since 49s) 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: overall HEALTH_OK 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:34.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:35.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:43:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: mon.a calling monitor election 2024-09-21T09:43:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: mon.b calling monitor election 2024-09-21T09:43:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:43:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: mon.c calling monitor election 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: monmap e3: 3 mons at {a=[v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0],b=[v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0],c=[v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: fsmap 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: osdmap e4: 0 total, 0 up, 0 in 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: mgrmap e13: y(active, since 49s) 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: overall HEALTH_OK 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:35.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:35.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:35.995 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:35 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:35 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:35 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:35 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:35 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:35 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:36.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:35 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:35 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:35 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:43:36.140 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:43:36.141 INFO:teuthology.orchestra.run.smithi105.stdout:{"epoch":3,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","modified":"2024-09-21T09:43:29.505849Z","created":"2024-09-21T09:41:48.442447Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3300","nonce":0},{"type":"v1","addr":"172.21.15.33:6789","nonce":0}]},"addr":"172.21.15.33:6789/0","public_addr":"172.21.15.33:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:3300","nonce":0},{"type":"v1","addr":"172.21.15.105:6789","nonce":0}]},"addr":"172.21.15.105:6789/0","public_addr":"172.21.15.105:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:3301","nonce":0},{"type":"v1","addr":"172.21.15.33:6790","nonce":0}]},"addr":"172.21.15.33:6790/0","public_addr":"172.21.15.33:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-21T09:43:36.141 INFO:teuthology.orchestra.run.smithi105.stderr:dumped monmap epoch 3 2024-09-21T09:43:36.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:36.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/1463649307' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:36.774 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:36.800 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-21T09:43:36.800 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph config generate-minimal-conf 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/1463649307' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.962 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.963 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/1463649307' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:36.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:37.510 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: Reconfiguring daemon mon.a on smithi033 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:37.738 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: Reconfiguring daemon mon.a on smithi033 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:37.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:37.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:37.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: Reconfiguring mon.a (unknown last config time)... 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: Reconfiguring daemon mon.a on smithi033 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:38.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:38 smithi033 ceph-mon[27969]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:38.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:38 smithi033 ceph-mon[27969]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:43:38.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:38 smithi033 ceph-mon[27969]: Reconfiguring daemon mon.c on smithi033 2024-09-21T09:43:38.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:38 smithi033 ceph-mon[39045]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:38.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:38 smithi033 ceph-mon[39045]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:43:38.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:38 smithi033 ceph-mon[39045]: Reconfiguring daemon mon.c on smithi033 2024-09-21T09:43:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:38 smithi105 ceph-mon[31286]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:38 smithi105 ceph-mon[31286]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:43:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:38 smithi105 ceph-mon[31286]: Reconfiguring daemon mon.c on smithi033 2024-09-21T09:43:40.190 INFO:teuthology.orchestra.run.smithi033.stdout:# minimal ceph.conf for a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:40.191 INFO:teuthology.orchestra.run.smithi033.stdout:[global] 2024-09-21T09:43:40.191 INFO:teuthology.orchestra.run.smithi033.stdout: fsid = a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:43:40.191 INFO:teuthology.orchestra.run.smithi033.stdout: mon_host = [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] 2024-09-21T09:43:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:43:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: Reconfiguring daemon mon.b on smithi105 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/780039469' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: Reconfiguring daemon mon.b on smithi105 2024-09-21T09:43:40.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:40 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/780039469' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:40.815 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-21T09:43:40.815 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:43:40.815 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:43:40.844 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:43:40.844 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:43:40.910 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:43:40.910 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.conf 2024-09-21T09:43:40.934 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:40.935 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:40.935 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:40.935 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:43:40.935 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:43:40.935 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:43:40.935 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:40.935 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: Reconfiguring daemon mon.b on smithi105 2024-09-21T09:43:40.935 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:40 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/780039469' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:40.940 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:43:40.940 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:43:41.005 INFO:tasks.cephadm:Adding mgr.y on smithi033 2024-09-21T09:43:41.005 INFO:tasks.cephadm:Adding mgr.x on smithi105 2024-09-21T09:43:41.006 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch apply mgr '2;smithi033=y;smithi105=x' 2024-09-21T09:43:41.282 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:43:42.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:42.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:42.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:42.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.351 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:42 smithi105 ceph-mon[31286]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:42.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[27969]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:42.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:42.498 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:42.498 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:42.498 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:42 smithi033 ceph-mon[39045]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:43.011 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled mgr update... 2024-09-21T09:43:43.569 DEBUG:teuthology.orchestra.run.smithi105:mgr.x> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x.service 2024-09-21T09:43:43.571 DEBUG:tasks.cephadm:set 0 configs 2024-09-21T09:43:43.571 INFO:tasks.cephadm:Deploying OSDs... 2024-09-21T09:43:43.572 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:43:43.572 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:43:43.592 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:43:43.592 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_1 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-09-21 09:43:28.953247010 +0000 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:43.652 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:43.653 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:43:43.721 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-09-21T09:43:43.721 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-09-21T09:43:43.721 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.00034589 s, 1.5 MB/s 2024-09-21T09:43:43.723 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:43:43.783 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_2 2024-09-21T09:43:43.841 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:43:43.841 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:43:43.841 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 974 Links: 1 2024-09-21T09:43:43.841 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:43:43.841 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:43:43.842 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-09-21 09:43:28.953247010 +0000 2024-09-21T09:43:43.842 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:43.842 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:43.842 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:43.842 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:43:43.909 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-09-21T09:43:43.909 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-09-21T09:43:43.909 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000230082 s, 2.2 MB/s 2024-09-21T09:43:43.910 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:43:43.969 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_3 2024-09-21T09:43:44.027 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:43:44.027 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:43:44.027 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 981 Links: 1 2024-09-21T09:43:44.028 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:43:44.028 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:43:44.028 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-09-21 09:43:28.954246994 +0000 2024-09-21T09:43:44.028 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-09-21 09:39:41.783856776 +0000 2024-09-21T09:43:44.028 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-09-21 09:39:41.783856776 +0000 2024-09-21T09:43:44.028 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-09-21 09:39:41.783856776 +0000 2024-09-21T09:43:44.028 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:43:44.094 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-09-21T09:43:44.094 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-09-21T09:43:44.094 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000212709 s, 2.4 MB/s 2024-09-21T09:43:44.095 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:43:44.152 DEBUG:teuthology.orchestra.run.smithi033:> stat /dev/vg_nvme/lv_4 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout:Device: 5h/5d Inode: 975 Links: 1 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout:Access: 2024-09-21 09:43:28.954246994 +0000 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout:Modify: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout:Change: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:44.210 INFO:teuthology.orchestra.run.smithi033.stdout: Birth: 2024-09-21 09:39:41.782856720 +0000 2024-09-21T09:43:44.211 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:43:44.273 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi033=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:44.273 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: Saving service mgr spec with placement smithi033=y;smithi105=x;count:2 2024-09-21T09:43:44.273 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: Deploying daemon mgr.x on smithi105 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[27969]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi033=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: Saving service mgr spec with placement smithi033=y;smithi105=x;count:2 2024-09-21T09:43:44.274 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: Deploying daemon mgr.x on smithi105 2024-09-21T09:43:44.275 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:44 smithi033 ceph-mon[39045]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:44.275 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records in 2024-09-21T09:43:44.276 INFO:teuthology.orchestra.run.smithi033.stderr:1+0 records out 2024-09-21T09:43:44.276 INFO:teuthology.orchestra.run.smithi033.stderr:512 bytes copied, 0.000212533 s, 2.4 MB/s 2024-09-21T09:43:44.277 DEBUG:teuthology.orchestra.run.smithi033:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi033=y;smithi105=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: Saving service mgr spec with placement smithi033=y;smithi105=x;count:2 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-21T09:43:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:43:44.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:44.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: Deploying daemon mgr.x on smithi105 2024-09-21T09:43:44.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:44 smithi105 ceph-mon[31286]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:44.334 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:43:44.334 DEBUG:teuthology.orchestra.run.smithi105:> dd if=/scratch_devs of=/dev/stdout 2024-09-21T09:43:44.350 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-21T09:43:44.351 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_1 2024-09-21T09:43:44.406 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-21T09:43:44.406 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:43:44.406 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-21T09:43:44.407 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:43:44.407 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:43:44.407 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-21 09:43:20.341595056 +0000 2024-09-21T09:43:44.407 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-21 09:39:41.779224393 +0000 2024-09-21T09:43:44.407 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-21 09:39:41.779224393 +0000 2024-09-21T09:43:44.407 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-21 09:39:41.779224393 +0000 2024-09-21T09:43:44.407 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-21T09:43:44.470 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-21T09:43:44.470 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-21T09:43:44.470 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000274825 s, 1.9 MB/s 2024-09-21T09:43:44.472 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-21T09:43:44.529 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_2 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 972 Links: 1 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-21 09:43:20.341595056 +0000 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:44.584 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:44.584 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-21T09:43:44.647 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-21T09:43:44.647 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-21T09:43:44.647 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000213177 s, 2.4 MB/s 2024-09-21T09:43:44.648 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-21T09:43:44.704 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_3 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-21 09:43:20.341595056 +0000 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:44.760 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:44.760 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-21T09:43:44.926 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-21T09:43:44.926 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-21T09:43:44.926 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.00026911 s, 1.9 MB/s 2024-09-21T09:43:44.927 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-21T09:43:44.942 DEBUG:teuthology.orchestra.run.smithi105:> stat /dev/vg_nvme/lv_4 2024-09-21T09:43:45.004 INFO:teuthology.orchestra.run.smithi105.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-21T09:43:45.004 INFO:teuthology.orchestra.run.smithi105.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-21T09:43:45.005 INFO:teuthology.orchestra.run.smithi105.stdout:Device: 5h/5d Inode: 974 Links: 1 2024-09-21T09:43:45.005 INFO:teuthology.orchestra.run.smithi105.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-21T09:43:45.005 INFO:teuthology.orchestra.run.smithi105.stdout:Context: system_u:object_r:device_t:s0 2024-09-21T09:43:45.005 INFO:teuthology.orchestra.run.smithi105.stdout:Access: 2024-09-21 09:43:44.880077809 +0000 2024-09-21T09:43:45.005 INFO:teuthology.orchestra.run.smithi105.stdout:Modify: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:45.005 INFO:teuthology.orchestra.run.smithi105.stdout:Change: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:45.005 INFO:teuthology.orchestra.run.smithi105.stdout: Birth: 2024-09-21 09:39:41.780224289 +0000 2024-09-21T09:43:45.005 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-21T09:43:45.110 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records in 2024-09-21T09:43:45.110 INFO:teuthology.orchestra.run.smithi105.stderr:1+0 records out 2024-09-21T09:43:45.110 INFO:teuthology.orchestra.run.smithi105.stderr:512 bytes copied, 0.000192933 s, 2.7 MB/s 2024-09-21T09:43:45.112 DEBUG:teuthology.orchestra.run.smithi105:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-21T09:43:45.127 INFO:tasks.cephadm:Deploying osd.0 on smithi033 with /dev/vg_nvme/lv_4... 2024-09-21T09:43:45.127 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:43:45.343 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 systemd[1]: Starting Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:43:45.387 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:45.611 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 podman[33351]: 2024-09-21 09:43:45.244547934 +0000 UTC m=+0.015233637 image pull fc3ec4a4f6be575045de0fd8a2fe84939e888152c447bdedb90add7841f7bef9 quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2 2024-09-21T09:43:45.611 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 podman[33351]: 2024-09-21 09:43:45.347736759 +0000 UTC m=+0.118422488 container create e1080a983fa6209fe2e4b292bbf5ad5b0bc8abea5497671886246e8b223ede9a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T09:43:45.611 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 podman[33351]: 2024-09-21 09:43:45.583527143 +0000 UTC m=+0.354212845 container init e1080a983fa6209fe2e4b292bbf5ad5b0bc8abea5497671886246e8b223ede9a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:43:45.612 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 podman[33351]: 2024-09-21 09:43:45.588181503 +0000 UTC m=+0.358867191 container start e1080a983fa6209fe2e4b292bbf5ad5b0bc8abea5497671886246e8b223ede9a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:43:45.612 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 ceph-mgr[33389]: -- 172.21.15.105:0/2370133000 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556d37afc340 con 0x556d37b08800 2024-09-21T09:43:45.957 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 bash[33351]: e1080a983fa6209fe2e4b292bbf5ad5b0bc8abea5497671886246e8b223ede9a 2024-09-21T09:43:45.958 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 systemd[1]: Started Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:43:45.958 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:45.700+0000 7fe3ff12eec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:43:45.958 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:45.760+0000 7fe3ff12eec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:43:45.958 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:45.826+0000 7fe3ff12eec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:43:45.958 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:45.955+0000 7fe3ff12eec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:43:46.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:46.102+0000 7fe3ff12eec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:43:46.547 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[27969]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:46.547 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.547 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.547 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.548 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.548 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:46.548 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[39045]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:46.548 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.548 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.548 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.548 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.548 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:46.655 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-mon[31286]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:46.655 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.656 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.656 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.656 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:46.656 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:46.656 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:46.375+0000 7fe3ff12eec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:43:46.656 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:46.439+0000 7fe3ff12eec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:43:46.656 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:46.609+0000 7fe3ff12eec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:43:47.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:47.459+0000 7fe3ff12eec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:43:47.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:47.522+0000 7fe3ff12eec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:43:47.903 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:43:47.927 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi033:vg_nvme/lv_4 2024-09-21T09:43:48.183 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: Reconfiguring daemon mgr.y on smithi033 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[27969]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.184 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: Reconfiguring daemon mgr.y on smithi033 2024-09-21T09:43:48.185 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:47 smithi033 ceph-mon[39045]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:48.221 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.221 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.221 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:48.221 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:48.222 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:48.222 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: Reconfiguring mgr.y (unknown last config time)... 2024-09-21T09:43:48.222 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:43:48.222 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:43:48.222 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:48.222 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: Reconfiguring daemon mgr.y on smithi033 2024-09-21T09:43:48.222 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:47 smithi105 ceph-mon[31286]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:48.507 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.219+0000 7fe3ff12eec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:43:48.508 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.329+0000 7fe3ff12eec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:43:48.508 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.447+0000 7fe3ff12eec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:43:48.623 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:43:48.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.505+0000 7fe3ff12eec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:43:48.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.629+0000 7fe3ff12eec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:43:48.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.694+0000 7fe3ff12eec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:43:48.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.751+0000 7fe3ff12eec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:43:49.150 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.885+0000 7fe3ff12eec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:43:49.150 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:48.948+0000 7fe3ff12eec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:43:49.150 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:49.012+0000 7fe3ff12eec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:43:49.150 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:49.091+0000 7fe3ff12eec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:43:49.150 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:49.147+0000 7fe3ff12eec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:43:49.530 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:49.203+0000 7fe3ff12eec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:43:50.133 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:50.133 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:50.133 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:50.133 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:50.133 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: Standby manager daemon x started 2024-09-21T09:43:50.133 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:43:50.133 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:43:50.133 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: Standby manager daemon x started 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:43:50.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:50 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:43:50.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:43:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:43:49.941+0000 7fe3ff12eec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:43:50.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:50.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:50.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:43:50.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:50.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: Standby manager daemon x started 2024-09-21T09:43:50.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:43:50.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:43:50.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:43:50.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:50 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/3248742146' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:43:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:43:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:51.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: mgrmap e14: y(active, since 64s), standbys: x 2024-09-21T09:43:51.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:43:51.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:51.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:43:51.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:51.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:43:51.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='client.14214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:43:51.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:43:51.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:43:51.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:52.198 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:52 smithi033 ceph-mon[27969]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:52.198 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:52 smithi033 ceph-mon[39045]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:52.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:52 smithi105 ceph-mon[31286]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:53.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1961808741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]: dispatch 2024-09-21T09:43:53.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]: dispatch 2024-09-21T09:43:53.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]': finished 2024-09-21T09:43:53.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[27969]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:43:53.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:43:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1961808741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]: dispatch 2024-09-21T09:43:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]: dispatch 2024-09-21T09:43:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]': finished 2024-09-21T09:43:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[39045]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:43:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:43:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:53 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1961808741' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]: dispatch 2024-09-21T09:43:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:53 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]: dispatch 2024-09-21T09:43:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:53 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27f5a40d-6f81-49ea-a8aa-0641d32c3667"}]': finished 2024-09-21T09:43:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:53 smithi105 ceph-mon[31286]: osdmap e5: 1 total, 0 up, 1 in 2024-09-21T09:43:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:43:54.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:54 smithi033 ceph-mon[27969]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:54.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:54 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/779187968' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:43:54.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:54 smithi033 ceph-mon[39045]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:54.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:54 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/779187968' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:43:54.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:54 smithi105 ceph-mon[31286]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:54.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:54 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/779187968' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:43:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:56 smithi033 ceph-mon[27969]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:56 smithi033 ceph-mon[39045]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:56 smithi105 ceph-mon[31286]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:58.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:58 smithi033 ceph-mon[27969]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:58.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:58 smithi033 ceph-mon[39045]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:58 smithi105 ceph-mon[31286]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:43:59.688 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:59 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:43:59.688 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:43:59 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:59.689 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:59 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:43:59.689 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:43:59 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:43:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:59 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-21T09:43:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:43:59 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:00.551 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:00 smithi033 ceph-mon[27969]: Deploying daemon osd.0 on smithi033 2024-09-21T09:44:00.551 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:00 smithi033 ceph-mon[27969]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:00.552 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:00 smithi033 ceph-mon[39045]: Deploying daemon osd.0 on smithi033 2024-09-21T09:44:00.552 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:00 smithi033 ceph-mon[39045]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:00 smithi105 ceph-mon[31286]: Deploying daemon osd.0 on smithi033 2024-09-21T09:44:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:00 smithi105 ceph-mon[31286]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:02.703 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:02 smithi033 ceph-mon[27969]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:02.703 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:02 smithi033 ceph-mon[39045]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:02 smithi105 ceph-mon[31286]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:04.658 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:04 smithi033 ceph-mon[27969]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:04.658 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:04.658 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:04.658 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:04.659 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:04 smithi033 ceph-mon[39045]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:04.659 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:04.659 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:04.659 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:04 smithi105 ceph-mon[31286]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:06.740 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:06 smithi033 ceph-mon[27969]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:06.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:06 smithi033 ceph-mon[39045]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:06 smithi105 ceph-mon[31286]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:07.396 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 0 on host 'smithi033' 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.958 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:07 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:07.981 DEBUG:teuthology.orchestra.run.smithi033:osd.0> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.0.service 2024-09-21T09:44:07.983 INFO:tasks.cephadm:Deploying osd.1 on smithi033 with /dev/vg_nvme/lv_3... 2024-09-21T09:44:07.983 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:07 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[27969]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[27969]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:44:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[27969]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:44:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[27969]: from='osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[27969]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[39045]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[39045]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:44:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[39045]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:44:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[39045]: from='osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-mon[39045]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:08.747 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[43436]: 2024-09-21T09:44:08.685+0000 7fe382cff640 -1 osd.0 0 waiting for initial osdmap 2024-09-21T09:44:08.748 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:44:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[43436]: 2024-09-21T09:44:08.687+0000 7fe37fb3a640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:44:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:08 smithi105 ceph-mon[31286]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:08 smithi105 ceph-mon[31286]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:44:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:08 smithi105 ceph-mon[31286]: osdmap e6: 1 total, 0 up, 1 in 2024-09-21T09:44:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:08 smithi105 ceph-mon[31286]: from='osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:08 smithi105 ceph-mon[31286]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:09.184 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:09 smithi033 ceph-mon[27969]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:09 smithi033 ceph-mon[27969]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:44:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:09 smithi033 ceph-mon[39045]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:09 smithi033 ceph-mon[39045]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:44:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:09 smithi105 ceph-mon[31286]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:09 smithi105 ceph-mon[31286]: osdmap e7: 1 total, 0 up, 1 in 2024-09-21T09:44:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[39045]: purged_snaps scrub starts 2024-09-21T09:44:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[39045]: purged_snaps scrub ok 2024-09-21T09:44:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[39045]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[39045]: osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496] boot 2024-09-21T09:44:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[39045]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:44:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[27969]: purged_snaps scrub starts 2024-09-21T09:44:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[27969]: purged_snaps scrub ok 2024-09-21T09:44:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[27969]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[27969]: osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496] boot 2024-09-21T09:44:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[27969]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:44:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:10 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:10 smithi105 ceph-mon[31286]: purged_snaps scrub starts 2024-09-21T09:44:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:10 smithi105 ceph-mon[31286]: purged_snaps scrub ok 2024-09-21T09:44:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:10 smithi105 ceph-mon[31286]: pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-21T09:44:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:10 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:10 smithi105 ceph-mon[31286]: osd.0 [v2:172.21.15.33:6802/3277300496,v1:172.21.15.33:6803/3277300496] boot 2024-09-21T09:44:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:10 smithi105 ceph-mon[31286]: osdmap e8: 1 total, 1 up, 1 in 2024-09-21T09:44:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:10 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:44:11.865 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[27969]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:44:11.865 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:11.865 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:11.865 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:11.865 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:11.865 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:11.865 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:11.865 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:11.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[39045]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:44:11.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:11.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:11.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:11.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:11.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:11.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:11.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:11 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:11 smithi105 ceph-mon[31286]: osdmap e9: 1 total, 1 up, 1 in 2024-09-21T09:44:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:11 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:11 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:11 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:11 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:11 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:11 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:11 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:12.523 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:44:12.543 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi033:vg_nvme/lv_3 2024-09-21T09:44:12.805 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:12.827 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:12 smithi033 ceph-mon[39045]: Detected new or changed devices on smithi033 2024-09-21T09:44:12.827 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:12 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi033 to 16068M 2024-09-21T09:44:12.827 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:12 smithi033 ceph-mon[39045]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:12.827 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:12 smithi033 ceph-mon[27969]: Detected new or changed devices on smithi033 2024-09-21T09:44:12.827 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:12 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi033 to 16068M 2024-09-21T09:44:12.827 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:12 smithi033 ceph-mon[27969]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:12 smithi105 ceph-mon[31286]: Detected new or changed devices on smithi033 2024-09-21T09:44:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:12 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi033 to 16068M 2024-09-21T09:44:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:12 smithi105 ceph-mon[31286]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:14.968 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:14 smithi033 ceph-mon[27969]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:14.968 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:14 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:44:14.968 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:14 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:44:14.968 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:14 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:14.969 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:14 smithi033 ceph-mon[39045]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:14.969 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:14 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:44:14.969 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:14 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:44:14.969 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:14 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:14 smithi105 ceph-mon[31286]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:14 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:44:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:14 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:44:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:14 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:15.924 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:15 smithi033 ceph-mon[27969]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:15.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:15 smithi033 ceph-mon[39045]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:15 smithi105 ceph-mon[31286]: from='client.24146 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:16 smithi033 ceph-mon[27969]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:16 smithi033 ceph-mon[39045]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:16 smithi105 ceph-mon[31286]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1211532383' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "120c7205-c87f-4d77-a255-14fff1e3b631"}]: dispatch 2024-09-21T09:44:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1211532383' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "120c7205-c87f-4d77-a255-14fff1e3b631"}]': finished 2024-09-21T09:44:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[27969]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:44:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3331067180' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:44:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1211532383' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "120c7205-c87f-4d77-a255-14fff1e3b631"}]: dispatch 2024-09-21T09:44:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1211532383' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "120c7205-c87f-4d77-a255-14fff1e3b631"}]': finished 2024-09-21T09:44:17.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[39045]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:44:17.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:17.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:17 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3331067180' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:44:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:17 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1211532383' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "120c7205-c87f-4d77-a255-14fff1e3b631"}]: dispatch 2024-09-21T09:44:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:17 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1211532383' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "120c7205-c87f-4d77-a255-14fff1e3b631"}]': finished 2024-09-21T09:44:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:17 smithi105 ceph-mon[31286]: osdmap e10: 2 total, 1 up, 2 in 2024-09-21T09:44:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:17 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3331067180' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:44:18.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:18 smithi033 ceph-mon[27969]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:18 smithi033 ceph-mon[39045]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:18 smithi105 ceph-mon[31286]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:19.990 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:19 smithi033 ceph-mon[39045]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:19.990 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:19 smithi033 ceph-mon[27969]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:20.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:19 smithi105 ceph-mon[31286]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:22.552 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:22 smithi033 ceph-mon[39045]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:22.552 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:22 smithi033 ceph-mon[27969]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:22 smithi105 ceph-mon[31286]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:23.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:23 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:44:23.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:23 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:23 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:44:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:23 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:23 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-21T09:44:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:23 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:24.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:24 smithi033 ceph-mon[27969]: Deploying daemon osd.1 on smithi033 2024-09-21T09:44:24.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:24 smithi033 ceph-mon[27969]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:24.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:24 smithi033 ceph-mon[39045]: Deploying daemon osd.1 on smithi033 2024-09-21T09:44:24.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:24 smithi033 ceph-mon[39045]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:24 smithi105 ceph-mon[31286]: Deploying daemon osd.1 on smithi033 2024-09-21T09:44:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:24 smithi105 ceph-mon[31286]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:26.619 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:26 smithi033 ceph-mon[27969]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:26.619 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:26 smithi033 ceph-mon[39045]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:26 smithi105 ceph-mon[31286]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:28 smithi033 ceph-mon[39045]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:28 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:28 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:28 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:28.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:28 smithi033 ceph-mon[27969]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:28.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:28 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:28.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:28 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:28.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:28 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:28 smithi105 ceph-mon[31286]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:28 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:28 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:28 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:30.692 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:30 smithi033 ceph-mon[27969]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:30.693 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:30 smithi033 ceph-mon[39045]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:30.693 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 1 on host 'smithi033' 2024-09-21T09:44:30.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:30 smithi105 ceph-mon[31286]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:31.295 DEBUG:teuthology.orchestra.run.smithi033:osd.1> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.1.service 2024-09-21T09:44:31.298 INFO:tasks.cephadm:Deploying osd.2 on smithi033 with /dev/vg_nvme/lv_2... 2024-09-21T09:44:31.298 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:44:31.604 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:31.980 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[27969]: from='osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[27969]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[39045]: from='osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[39045]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:31.981 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:31 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:31 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:31 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:31 smithi105 ceph-mon[31286]: from='osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:44:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:31 smithi105 ceph-mon[31286]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-21T09:44:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:31 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:31 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[27969]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[27969]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:44:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[27969]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:44:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[27969]: from='osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[27969]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:32.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[39045]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:32.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[39045]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:44:32.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[39045]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:44:32.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:32.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[39045]: from='osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:32.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-mon[39045]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:32.997 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[49350]: 2024-09-21T09:44:32.715+0000 7fd2d26d8640 -1 osd.1 0 waiting for initial osdmap 2024-09-21T09:44:32.997 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:44:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[49350]: 2024-09-21T09:44:32.717+0000 7fd2cf513640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:44:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:32 smithi105 ceph-mon[31286]: pgmap v53: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:32 smithi105 ceph-mon[31286]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:44:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:32 smithi105 ceph-mon[31286]: osdmap e11: 2 total, 1 up, 2 in 2024-09-21T09:44:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:32 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:32 smithi105 ceph-mon[31286]: from='osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:32 smithi105 ceph-mon[31286]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:33.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:33 smithi033 ceph-mon[39045]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:33.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:33 smithi033 ceph-mon[39045]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:44:33.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:33 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:33.903 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:33 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:33.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:33 smithi033 ceph-mon[27969]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:33.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:33 smithi033 ceph-mon[27969]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:44:33.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:33 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:33.903 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:33 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:33 smithi105 ceph-mon[31286]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:33 smithi105 ceph-mon[31286]: osdmap e12: 2 total, 1 up, 2 in 2024-09-21T09:44:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:33 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:33 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[39045]: purged_snaps scrub starts 2024-09-21T09:44:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[39045]: purged_snaps scrub ok 2024-09-21T09:44:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[39045]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[39045]: osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417] boot 2024-09-21T09:44:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[39045]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:44:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[27969]: purged_snaps scrub starts 2024-09-21T09:44:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[27969]: purged_snaps scrub ok 2024-09-21T09:44:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[27969]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[27969]: osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417] boot 2024-09-21T09:44:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[27969]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:44:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:34 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:35.011 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:44:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:34 smithi105 ceph-mon[31286]: purged_snaps scrub starts 2024-09-21T09:44:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:34 smithi105 ceph-mon[31286]: purged_snaps scrub ok 2024-09-21T09:44:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:34 smithi105 ceph-mon[31286]: pgmap v56: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-21T09:44:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:34 smithi105 ceph-mon[31286]: osd.1 [v2:172.21.15.33:6810/2439650417,v1:172.21.15.33:6811/2439650417] boot 2024-09-21T09:44:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:34 smithi105 ceph-mon[31286]: osdmap e13: 2 total, 2 up, 2 in 2024-09-21T09:44:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:34 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:44:35.036 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi033:vg_nvme/lv_2 2024-09-21T09:44:35.503 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:35.936 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:35 smithi033 ceph-mon[39045]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:44:35.936 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:35 smithi033 ceph-mon[39045]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:35.936 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:35 smithi033 ceph-mon[27969]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:44:35.936 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:35 smithi033 ceph-mon[27969]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:35 smithi105 ceph-mon[31286]: osdmap e14: 2 total, 2 up, 2 in 2024-09-21T09:44:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:35 smithi105 ceph-mon[31286]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: Detected new or changed devices on smithi033 2024-09-21T09:44:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi033 to 8034M 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[27969]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: Detected new or changed devices on smithi033 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi033 to 8034M 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:37 smithi033 ceph-mon[39045]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: Detected new or changed devices on smithi033 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi033 to 8034M 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:38.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:38.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:44:38.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:38.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:37 smithi105 ceph-mon[31286]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:38.901 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:38 smithi033 ceph-mon[39045]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:38.901 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:38 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:44:38.902 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:38 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:44:38.902 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:38 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:38.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:38 smithi033 ceph-mon[27969]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:38.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:38 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:44:38.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:38 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:44:38.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:38 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:38 smithi105 ceph-mon[31286]: from='client.14268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:44:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:38 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:44:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:38 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:44:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:38 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:39 smithi033 ceph-mon[39045]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:40.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:39 smithi033 ceph-mon[27969]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:40.279 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:39 smithi105 ceph-mon[31286]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3642568792' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bf7f6b7d-0134-4ce6-b776-040aa14b9075"}]: dispatch 2024-09-21T09:44:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3642568792' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bf7f6b7d-0134-4ce6-b776-040aa14b9075"}]': finished 2024-09-21T09:44:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[39045]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:44:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2831642795' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:44:41.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3642568792' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bf7f6b7d-0134-4ce6-b776-040aa14b9075"}]: dispatch 2024-09-21T09:44:41.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3642568792' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bf7f6b7d-0134-4ce6-b776-040aa14b9075"}]': finished 2024-09-21T09:44:41.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[27969]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:44:41.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:41.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:40 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2831642795' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:44:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:40 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3642568792' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bf7f6b7d-0134-4ce6-b776-040aa14b9075"}]: dispatch 2024-09-21T09:44:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:40 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3642568792' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bf7f6b7d-0134-4ce6-b776-040aa14b9075"}]': finished 2024-09-21T09:44:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:40 smithi105 ceph-mon[31286]: osdmap e15: 3 total, 2 up, 3 in 2024-09-21T09:44:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:40 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2831642795' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:44:42.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:41 smithi033 ceph-mon[27969]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:42.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:41 smithi033 ceph-mon[39045]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:41 smithi105 ceph-mon[31286]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:44.600 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:44 smithi033 ceph-mon[39045]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:44.600 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:44 smithi033 ceph-mon[27969]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:44 smithi105 ceph-mon[31286]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:46 smithi033 ceph-mon[27969]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:44:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:46.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:46 smithi033 ceph-mon[39045]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:46.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:44:46.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:46 smithi105 ceph-mon[31286]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-21T09:44:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:44:47.704 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:47 smithi033 ceph-mon[39045]: Deploying daemon osd.2 on smithi033 2024-09-21T09:44:47.705 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:47 smithi033 ceph-mon[27969]: Deploying daemon osd.2 on smithi033 2024-09-21T09:44:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:47 smithi105 ceph-mon[31286]: Deploying daemon osd.2 on smithi033 2024-09-21T09:44:48.459 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:48 smithi033 ceph-mon[27969]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:48.459 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:48 smithi033 ceph-mon[39045]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:48 smithi105 ceph-mon[31286]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:50.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:50 smithi033 ceph-mon[39045]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:50 smithi033 ceph-mon[27969]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:50 smithi105 ceph-mon[31286]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:51.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:51.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:51.734 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:51 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:51.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:51.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:51.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:51 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:44:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:51 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:52.830 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:52 smithi033 ceph-mon[27969]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:52.831 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:52 smithi033 ceph-mon[39045]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:52 smithi105 ceph-mon[31286]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:53.780 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 2 on host 'smithi033' 2024-09-21T09:44:54.373 DEBUG:teuthology.orchestra.run.smithi033:osd.2> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.2.service 2024-09-21T09:44:54.375 INFO:tasks.cephadm:Deploying osd.3 on smithi033 with /dev/vg_nvme/lv_1... 2024-09-21T09:44:54.375 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:44:54.679 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:54.937 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[39045]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:54.937 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:54.937 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[39045]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[27969]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[27969]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:54.938 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:54 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:54 smithi105 ceph-mon[31286]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:54 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:54 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:54 smithi105 ceph-mon[31286]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-21T09:44:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:54 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:55.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:54 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[39045]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[39045]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[39045]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[39045]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[27969]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[27969]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[27969]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:56.085 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-mon[27969]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:56.086 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[55754]: 2024-09-21T09:44:55.790+0000 7ff4e78ff640 -1 osd.2 0 waiting for initial osdmap 2024-09-21T09:44:56.086 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:44:55 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[55754]: 2024-09-21T09:44:55.794+0000 7ff4dff0f640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:44:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:55 smithi105 ceph-mon[31286]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:44:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:55 smithi105 ceph-mon[31286]: osdmap e16: 3 total, 2 up, 3 in 2024-09-21T09:44:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:55 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:55 smithi105 ceph-mon[31286]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:44:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:55 smithi105 ceph-mon[31286]: pgmap v71: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-21T09:44:57.038 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:44:57.056 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:56 smithi033 ceph-mon[39045]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:57.056 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:56 smithi033 ceph-mon[39045]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:44:57.056 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:56 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:57.056 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:56 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:57.056 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:56 smithi033 ceph-mon[27969]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:57.056 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:56 smithi033 ceph-mon[27969]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:44:57.056 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:56 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:57.056 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:56 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:57.057 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi033:vg_nvme/lv_1 2024-09-21T09:44:57.181 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:56 smithi105 ceph-mon[31286]: from='osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:44:57.181 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:56 smithi105 ceph-mon[31286]: osdmap e17: 3 total, 2 up, 3 in 2024-09-21T09:44:57.181 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:56 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:57.181 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:56 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:57.865 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:44:57.988 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[39045]: purged_snaps scrub starts 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[39045]: purged_snaps scrub ok 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[39045]: osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464] boot 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[39045]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[27969]: purged_snaps scrub starts 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[27969]: purged_snaps scrub ok 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[27969]: osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464] boot 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[27969]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:57.989 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:44:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:57 smithi105 ceph-mon[31286]: purged_snaps scrub starts 2024-09-21T09:44:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:57 smithi105 ceph-mon[31286]: purged_snaps scrub ok 2024-09-21T09:44:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:57 smithi105 ceph-mon[31286]: osd.2 [v2:172.21.15.33:6818/3305661464,v1:172.21.15.33:6819/3305661464] boot 2024-09-21T09:44:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:57 smithi105 ceph-mon[31286]: osdmap e18: 3 total, 3 up, 3 in 2024-09-21T09:44:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:44:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-21T09:44:58.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[39045]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:44:58.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:44:58.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[39045]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:44:58.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:44:58.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:44:58.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[39045]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:44:58.876 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[27969]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:44:58.876 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:44:58.876 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[27969]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:44:58.876 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:44:58.876 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:44:58.876 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:58 smithi033 ceph-mon[27969]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:44:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:58 smithi105 ceph-mon[31286]: pgmap v74: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:44:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:58 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-21T09:44:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:58 smithi105 ceph-mon[31286]: osdmap e19: 3 total, 3 up, 3 in 2024-09-21T09:44:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:58 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:44:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:58 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:44:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:58 smithi105 ceph-mon[31286]: osdmap e20: 3 total, 3 up, 3 in 2024-09-21T09:44:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:44:59 smithi033 ceph-mon[27969]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:44:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:44:59 smithi033 ceph-mon[39045]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:44:59.996 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:44:59 smithi033 sudo[60124]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:44:59.996 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:44:59 smithi033 sudo[60124]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:44:59.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:44:59 smithi033 sudo[60124]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:44:59.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:44:59 smithi033 sudo[60124]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:44:59 smithi105 ceph-mon[31286]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:00.496 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:45:00 smithi033 sudo[60164]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:45:00.496 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:45:00 smithi033 sudo[60164]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:00.496 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:45:00 smithi033 sudo[60164]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:00.496 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:45:00 smithi033 sudo[60164]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:00.841 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:45:00 smithi033 sudo[60307]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:45:00.841 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:45:00 smithi033 sudo[60307]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:00.841 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:45:00 smithi033 sudo[60307]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:00.842 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:45:00 smithi033 sudo[60307]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:01.150 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:45:01.150 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:45:01.150 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: Detected new or changed devices on smithi033 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi033 to 5356M 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.151 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:01.152 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: Detected new or changed devices on smithi033 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi033 to 5356M 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:01.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:00 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-21T09:45:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: osdmap e21: 3 total, 3 up, 3 in 2024-09-21T09:45:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi033:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: Detected new or changed devices on smithi033 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi033 to 5356M 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:01.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:00 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:01.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:45:01 smithi033 sudo[60411]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:45:01.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:45:01 smithi033 sudo[60411]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:01.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:45:01 smithi033 sudo[60411]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:01.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:45:01 smithi033 sudo[60411]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:01.895 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:01 smithi033 ceph-mon[39045]: pgmap v79: 1 pgs: 1 active+clean; 320 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:01.895 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:45:01 smithi033 sudo[60421]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:45:01.895 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:45:01 smithi033 sudo[60421]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:01.895 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:45:01 smithi033 sudo[60421]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:01.895 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:45:01 smithi033 sudo[60421]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:01.896 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:01 smithi033 ceph-mon[27969]: pgmap v79: 1 pgs: 1 active+clean; 320 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:02.172 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60445]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:45:02.172 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60445]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:02.172 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60445]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:02.173 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60445]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:01 smithi105 ceph-mon[31286]: pgmap v79: 1 pgs: 1 active+clean; 320 KiB data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:02.981 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60490]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:45:02.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60490]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:02.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60490]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:02.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60490]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:03.235 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60519]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:45:03.235 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60519]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:03.235 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60519]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:03.235 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:02 smithi033 sudo[60519]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:03.483 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 sudo[34052]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:45:03.483 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 sudo[34052]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:03.483 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 sudo[34052]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:03.483 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 sudo[34052]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:03.489 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/4017291791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]': finished 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:03.490 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/4017291791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]: dispatch 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]: dispatch 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]': finished 2024-09-21T09:45:03.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:45:03.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:03.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:03.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:03.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 sudo[34055]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 sudo[34055]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 sudo[34055]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 sudo[34055]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/4017291791' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]: dispatch 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]: dispatch 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "697c4c8e-f54d-4cfe-8dfd-a7c591a94a87"}]': finished 2024-09-21T09:45:03.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: osdmap e22: 4 total, 3 up, 4 in 2024-09-21T09:45:03.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:03.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:03.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:03.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:03 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:04.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 sudo[60603]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-21T09:45:04.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 sudo[60603]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:04.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 sudo[60603]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:04.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:03 smithi033 sudo[60603]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:04.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:04.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: pgmap v81: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:04.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:04.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:04.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2387543978' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 sudo[60791]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 sudo[60791]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 sudo[60791]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 sudo[60791]: pam_unix(sudo:session): session closed for user root 2024-09-21T09:45:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: pgmap v81: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2387543978' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:04 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: pgmap v81: 1 pgs: 1 active+clean; 320 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2387543978' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:04 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-21T09:45:06.514 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:06 smithi033 ceph-mon[27969]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.9 KiB/s rd, 237 KiB/s wr, 16 op/s 2024-09-21T09:45:06.514 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:06 smithi033 ceph-mon[39045]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.9 KiB/s rd, 237 KiB/s wr, 16 op/s 2024-09-21T09:45:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:06 smithi105 ceph-mon[31286]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.9 KiB/s rd, 237 KiB/s wr, 16 op/s 2024-09-21T09:45:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:08 smithi033 ceph-mon[27969]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.6 KiB/s rd, 174 KiB/s wr, 11 op/s 2024-09-21T09:45:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:08 smithi033 ceph-mon[39045]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.6 KiB/s rd, 174 KiB/s wr, 11 op/s 2024-09-21T09:45:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:08 smithi105 ceph-mon[31286]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.6 KiB/s rd, 174 KiB/s wr, 11 op/s 2024-09-21T09:45:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:45:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:09.794 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:45:09.794 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:09.795 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-21T09:45:09.795 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:10.768 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:10 smithi033 ceph-mon[27969]: Deploying daemon osd.3 on smithi033 2024-09-21T09:45:10.768 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:10 smithi033 ceph-mon[27969]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.8 KiB/s rd, 137 KiB/s wr, 9 op/s 2024-09-21T09:45:10.768 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:10 smithi033 ceph-mon[39045]: Deploying daemon osd.3 on smithi033 2024-09-21T09:45:10.768 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:10 smithi033 ceph-mon[39045]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.8 KiB/s rd, 137 KiB/s wr, 9 op/s 2024-09-21T09:45:10.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:10 smithi105 ceph-mon[31286]: Deploying daemon osd.3 on smithi033 2024-09-21T09:45:10.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:10 smithi105 ceph-mon[31286]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.8 KiB/s rd, 137 KiB/s wr, 9 op/s 2024-09-21T09:45:12.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:12 smithi033 ceph-mon[39045]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-21T09:45:12.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:12 smithi033 ceph-mon[27969]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-21T09:45:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:12 smithi105 ceph-mon[31286]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2024-09-21T09:45:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:14 smithi033 ceph-mon[39045]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.6 KiB/s rd, 126 KiB/s wr, 8 op/s 2024-09-21T09:45:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:14 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:14 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:14 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:14 smithi033 ceph-mon[27969]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.6 KiB/s rd, 126 KiB/s wr, 8 op/s 2024-09-21T09:45:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:14 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:14 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:14 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:14 smithi105 ceph-mon[31286]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.6 KiB/s rd, 126 KiB/s wr, 8 op/s 2024-09-21T09:45:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:14 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:14 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:14 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:16.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:16 smithi033 ceph-mon[27969]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-21T09:45:16.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:16 smithi033 ceph-mon[39045]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-21T09:45:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:16 smithi105 ceph-mon[31286]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2024-09-21T09:45:17.524 INFO:teuthology.orchestra.run.smithi033.stdout:Created osd(s) 3 on host 'smithi033' 2024-09-21T09:45:18.126 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.127 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:17 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.155 DEBUG:teuthology.orchestra.run.smithi033:osd.3> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.3.service 2024-09-21T09:45:18.157 INFO:tasks.cephadm:Deploying osd.4 on smithi105 with /dev/vg_nvme/lv_4... 2024-09-21T09:45:18.157 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_4 2024-09-21T09:45:18.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:18.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:18.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:45:18.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-21T09:45:18.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:18.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:18.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:17 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:18.407 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:45:19.225 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[39045]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:45:19.225 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[39045]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:45:19.225 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:19.225 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[39045]: from='osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:45:19.225 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[39045]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:45:19.225 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[27969]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:45:19.225 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[27969]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:45:19.226 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:19.226 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[27969]: from='osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:45:19.226 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-mon[27969]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:45:19.226 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[62324]: 2024-09-21T09:45:18.845+0000 7fe81068c640 -1 osd.3 0 waiting for initial osdmap 2024-09-21T09:45:19.226 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:45:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[62324]: 2024-09-21T09:45:18.849+0000 7fe80b4c3640 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:45:19.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:18 smithi105 ceph-mon[31286]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:45:19.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:18 smithi105 ceph-mon[31286]: osdmap e23: 4 total, 3 up, 4 in 2024-09-21T09:45:19.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:18 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:19.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:18 smithi105 ceph-mon[31286]: from='osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:45:19.249 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:18 smithi105 ceph-mon[31286]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]: dispatch 2024-09-21T09:45:20.078 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:19 smithi105 ceph-mon[31286]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:45:20.078 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:19 smithi105 ceph-mon[31286]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:45:20.078 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:19 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:20.078 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:19 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:20.078 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:19 smithi105 ceph-mon[31286]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[39045]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[39045]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[39045]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[27969]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]}]': finished 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[27969]: osdmap e24: 4 total, 3 up, 4 in 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:20.200 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:19 smithi033 ceph-mon[27969]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-21T09:45:20.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:20 smithi105 ceph-mon[31286]: purged_snaps scrub starts 2024-09-21T09:45:20.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:20 smithi105 ceph-mon[31286]: purged_snaps scrub ok 2024-09-21T09:45:20.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:20 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:20.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:20 smithi105 ceph-mon[31286]: osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171] boot 2024-09-21T09:45:20.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:20 smithi105 ceph-mon[31286]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:45:20.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:20 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:20.920 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:45:20.940 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi105:vg_nvme/lv_4 2024-09-21T09:45:21.215 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[27969]: purged_snaps scrub starts 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[27969]: purged_snaps scrub ok 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[27969]: osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171] boot 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[27969]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[39045]: purged_snaps scrub starts 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[39045]: purged_snaps scrub ok 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:21.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[39045]: osd.3 [v2:172.21.15.33:6826/510343171,v1:172.21.15.33:6827/510343171] boot 2024-09-21T09:45:21.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[39045]: osdmap e25: 4 total, 4 up, 4 in 2024-09-21T09:45:21.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:20 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:45:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: Detected new or changed devices on smithi033 2024-09-21T09:45:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi033 to 4017M 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:21 smithi105 ceph-mon[31286]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:22.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: Detected new or changed devices on smithi033 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi033 to 4017M 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[27969]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: Detected new or changed devices on smithi033 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: osdmap e26: 4 total, 4 up, 4 in 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi033 to 4017M 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:22.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:21 smithi033 ceph-mon[39045]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:23 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:23 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:23 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:23.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:23 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:23.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:23 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:23.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:23 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:23 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:23 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:23 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:24.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:24 smithi033 ceph-mon[27969]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:24.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:24 smithi033 ceph-mon[27969]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:24.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:24 smithi033 ceph-mon[39045]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:24.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:24 smithi033 ceph-mon[39045]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:24.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:24 smithi105 ceph-mon[31286]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:24.529 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:24 smithi105 ceph-mon[31286]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/3202794139' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]: dispatch 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]: dispatch 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]': finished 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[27969]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/3202794139' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]: dispatch 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]: dispatch 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]': finished 2024-09-21T09:45:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[39045]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:45:25.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:25 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:25 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/3202794139' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]: dispatch 2024-09-21T09:45:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:25 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]: dispatch 2024-09-21T09:45:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:25 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3040cdba-b341-48d0-bdf0-c6b6655fb164"}]': finished 2024-09-21T09:45:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:25 smithi105 ceph-mon[31286]: osdmap e27: 5 total, 4 up, 5 in 2024-09-21T09:45:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:25 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:26.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:26 smithi033 ceph-mon[27969]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:26.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:26 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/2661901703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:26 smithi033 ceph-mon[39045]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:26 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/2661901703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:26.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:26 smithi105 ceph-mon[31286]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:26 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/2661901703' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:28.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:28 smithi033 ceph-mon[27969]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:28 smithi033 ceph-mon[39045]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:28.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:28 smithi105 ceph-mon[31286]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:30.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:30 smithi105 ceph-mon[31286]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:30.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:30 smithi033 ceph-mon[27969]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:30 smithi033 ceph-mon[39045]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:32.673 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:32 smithi105 ceph-mon[31286]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:32.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:32 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:45:32.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:32 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:32.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:32 smithi105 ceph-mon[31286]: Deploying daemon osd.4 on smithi105 2024-09-21T09:45:32.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:32 smithi033 ceph-mon[39045]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:32.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:32 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:45:32.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:32 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:32.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:32 smithi033 ceph-mon[39045]: Deploying daemon osd.4 on smithi105 2024-09-21T09:45:32.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:32 smithi033 ceph-mon[27969]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:32.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:32 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-21T09:45:32.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:32 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:32.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:32 smithi033 ceph-mon[27969]: Deploying daemon osd.4 on smithi105 2024-09-21T09:45:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:34 smithi033 ceph-mon[39045]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:34.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:34 smithi033 ceph-mon[27969]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:34.766 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:34 smithi105 ceph-mon[31286]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:36.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:36 smithi033 ceph-mon[27969]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:36 smithi033 ceph-mon[39045]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:36 smithi105 ceph-mon[31286]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:37 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:37 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:37.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:37.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:37.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:37 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:38.566 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:38 smithi105 ceph-mon[31286]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:38 smithi033 ceph-mon[27969]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:38 smithi033 ceph-mon[39045]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:39.906 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 4 on host 'smithi105' 2024-09-21T09:45:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[39045]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[39045]: from='osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:45:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[39045]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:45:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[27969]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[27969]: from='osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:45:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:39 smithi033 ceph-mon[27969]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:45:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:39 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:39 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:39 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:39 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:39 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:39 smithi105 ceph-mon[31286]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:39 smithi105 ceph-mon[31286]: from='osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:45:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:39 smithi105 ceph-mon[31286]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-21T09:45:40.582 DEBUG:teuthology.orchestra.run.smithi105:osd.4> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.4.service 2024-09-21T09:45:40.584 INFO:tasks.cephadm:Deploying osd.5 on smithi105 with /dev/vg_nvme/lv_3... 2024-09-21T09:45:40.584 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_3 2024-09-21T09:45:41.147 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:41.147 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[39045]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[39045]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[39045]: from='osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[39045]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[27969]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[27969]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:45:41.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:41.149 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[27969]: from='osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:45:41.149 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:40 smithi033 ceph-mon[27969]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:45:41.243 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:45:41.263 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:45:41.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:41.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:41.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:40 smithi105 ceph-mon[31286]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:45:41.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:40 smithi105 ceph-mon[31286]: osdmap e28: 5 total, 4 up, 5 in 2024-09-21T09:45:41.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:40 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:41.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:40 smithi105 ceph-mon[31286]: from='osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:45:41.264 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:40 smithi105 ceph-mon[31286]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:45:41.530 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:45:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[36605]: 2024-09-21T09:45:41.260+0000 7f21fcf34640 -1 osd.4 0 waiting for initial osdmap 2024-09-21T09:45:41.530 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:45:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[36605]: 2024-09-21T09:45:41.265+0000 7f21f6d69640 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:45:42.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:42 smithi105 ceph-mon[31286]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:45:42.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:42 smithi105 ceph-mon[31286]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:45:42.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:42.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:42.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:42 smithi105 ceph-mon[31286]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:42.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:42 smithi105 ceph-mon[31286]: osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360] boot 2024-09-21T09:45:42.430 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:42 smithi105 ceph-mon[31286]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:45:42.431 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:42 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:42.446 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[39045]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:45:42.446 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[39045]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:45:42.446 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:42.446 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[39045]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[39045]: osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360] boot 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[39045]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[27969]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[27969]: osdmap e29: 5 total, 4 up, 5 in 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[27969]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[27969]: osd.4 [v2:172.21.15.105:6800/1234622360,v1:172.21.15.105:6801/1234622360] boot 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[27969]: osdmap e30: 5 total, 5 up, 5 in 2024-09-21T09:45:42.447 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:42 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:45:43.657 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:43 smithi105 ceph-mon[31286]: purged_snaps scrub starts 2024-09-21T09:45:43.658 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:43 smithi105 ceph-mon[31286]: purged_snaps scrub ok 2024-09-21T09:45:43.658 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:43 smithi105 ceph-mon[31286]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:45:43.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:43 smithi033 ceph-mon[39045]: purged_snaps scrub starts 2024-09-21T09:45:43.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:43 smithi033 ceph-mon[39045]: purged_snaps scrub ok 2024-09-21T09:45:43.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:43 smithi033 ceph-mon[39045]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:45:43.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:43 smithi033 ceph-mon[27969]: purged_snaps scrub starts 2024-09-21T09:45:43.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:43 smithi033 ceph-mon[27969]: purged_snaps scrub ok 2024-09-21T09:45:43.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:43 smithi033 ceph-mon[27969]: osdmap e31: 5 total, 5 up, 5 in 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:44.663 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:44 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:45:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:45:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: osdmap e32: 5 total, 5 up, 5 in 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:45:44.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:44 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:45:45.402 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:45:45.423 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi105:vg_nvme/lv_3 2024-09-21T09:45:45.660 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:45:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:45 smithi033 ceph-mon[27969]: Detected new or changed devices on smithi105 2024-09-21T09:45:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:45 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi105 to 17088M 2024-09-21T09:45:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:45 smithi033 ceph-mon[39045]: Detected new or changed devices on smithi105 2024-09-21T09:45:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:45 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi105 to 17088M 2024-09-21T09:45:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:45 smithi105 ceph-mon[31286]: Detected new or changed devices on smithi105 2024-09-21T09:45:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:45 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi105 to 17088M 2024-09-21T09:45:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:46 smithi033 ceph-mon[27969]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-09-21T09:45:46.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:46 smithi033 ceph-mon[39045]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-09-21T09:45:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:46 smithi105 ceph-mon[31286]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 141 KiB/s, 0 objects/s recovering 2024-09-21T09:45:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[27969]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:45:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[27969]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[39045]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:45:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[39045]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:48 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:48 smithi105 ceph-mon[31286]: pgmap v113: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:45:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:48 smithi105 ceph-mon[31286]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:45:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:48 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:45:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:48 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:45:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:48 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:50.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:50 smithi105 ceph-mon[31286]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:45:50.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:50 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/3391790803' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]: dispatch 2024-09-21T09:45:50.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:50 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]: dispatch 2024-09-21T09:45:50.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:50 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]': finished 2024-09-21T09:45:50.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:50 smithi105 ceph-mon[31286]: osdmap e33: 6 total, 5 up, 6 in 2024-09-21T09:45:50.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:50 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:45:50.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:50 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/310144427' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[27969]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:45:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/3391790803' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]: dispatch 2024-09-21T09:45:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]: dispatch 2024-09-21T09:45:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]': finished 2024-09-21T09:45:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[27969]: osdmap e33: 6 total, 5 up, 6 in 2024-09-21T09:45:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:45:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/310144427' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[39045]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:45:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/3391790803' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]: dispatch 2024-09-21T09:45:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]: dispatch 2024-09-21T09:45:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e621bf2-e921-45ab-8ee8-18b44317c562"}]': finished 2024-09-21T09:45:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[39045]: osdmap e33: 6 total, 5 up, 6 in 2024-09-21T09:45:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:45:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:50 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/310144427' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:45:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:52 smithi033 ceph-mon[27969]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:45:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:52 smithi033 ceph-mon[39045]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:45:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:52 smithi105 ceph-mon[31286]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:45:54.476 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:54 smithi105 ceph-mon[31286]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:45:54.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:54 smithi033 ceph-mon[27969]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:45:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:54 smithi033 ceph-mon[39045]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:45:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:56 smithi033 ceph-mon[39045]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:45:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:56 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:45:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:56 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:56.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:56 smithi033 ceph-mon[27969]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:45:56.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:56 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:45:56.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:56 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:56 smithi105 ceph-mon[31286]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:45:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:56 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-21T09:45:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:56 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:45:57.674 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:57 smithi105 ceph-mon[31286]: Deploying daemon osd.5 on smithi105 2024-09-21T09:45:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:57 smithi033 ceph-mon[27969]: Deploying daemon osd.5 on smithi105 2024-09-21T09:45:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:57 smithi033 ceph-mon[39045]: Deploying daemon osd.5 on smithi105 2024-09-21T09:45:58.528 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:45:58 smithi105 ceph-mon[31286]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:45:58.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:45:58 smithi033 ceph-mon[27969]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:45:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:45:58 smithi033 ceph-mon[39045]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:00.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:00 smithi033 ceph-mon[27969]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:00.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:00 smithi033 ceph-mon[39045]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:00.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:00 smithi105 ceph-mon[31286]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:01 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:01.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:01 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:01.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:01 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:01.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:01 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:01 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:01 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:01 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:02.741 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:02 smithi105 ceph-mon[31286]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:02 smithi033 ceph-mon[27969]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:02 smithi033 ceph-mon[39045]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:04.094 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 5 on host 'smithi105' 2024-09-21T09:46:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:04 smithi105 ceph-mon[31286]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:04 smithi105 ceph-mon[31286]: from='osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:46:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:04 smithi105 ceph-mon[31286]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:46:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:04 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.819 DEBUG:teuthology.orchestra.run.smithi105:osd.5> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.5.service 2024-09-21T09:46:04.821 INFO:tasks.cephadm:Deploying osd.6 on smithi105 with /dev/vg_nvme/lv_2... 2024-09-21T09:46:04.821 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_2 2024-09-21T09:46:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[27969]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[27969]: from='osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:46:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[27969]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:46:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[39045]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:04.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[39045]: from='osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:46:04.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[39045]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-21T09:46:04.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:04.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:04 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:05.543 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:46:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[39045]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:46:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[39045]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:46:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[39045]: from='osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[39045]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:05.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[27969]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:46:05.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[27969]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:46:05.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:05.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[27969]: from='osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:05.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:05 smithi033 ceph-mon[27969]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:05 smithi105 ceph-mon[31286]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:46:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:05 smithi105 ceph-mon[31286]: osdmap e34: 6 total, 5 up, 6 in 2024-09-21T09:46:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:05 smithi105 ceph-mon[31286]: from='osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:05 smithi105 ceph-mon[31286]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:06.030 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:46:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[42977]: 2024-09-21T09:46:05.707+0000 7f0116b0d640 -1 osd.5 0 waiting for initial osdmap 2024-09-21T09:46:06.030 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:46:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[42977]: 2024-09-21T09:46:05.711+0000 7f0110942640 -1 osd.5 35 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:46:06.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:06 smithi105 ceph-mon[31286]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:06.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:06 smithi105 ceph-mon[31286]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:06.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:06 smithi105 ceph-mon[31286]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:46:06.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:06 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:06 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:06 smithi105 ceph-mon[31286]: osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054] boot 2024-09-21T09:46:06.969 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:06 smithi105 ceph-mon[31286]: osdmap e36: 6 total, 6 up, 6 in 2024-09-21T09:46:06.970 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:06 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[39045]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[39045]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[39045]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:46:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[39045]: osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054] boot 2024-09-21T09:46:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[39045]: osdmap e36: 6 total, 6 up, 6 in 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[27969]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[27969]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[27969]: osdmap e35: 6 total, 5 up, 6 in 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[27969]: osd.5 [v2:172.21.15.105:6808/1890006054,v1:172.21.15.105:6809/1890006054] boot 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[27969]: osdmap e36: 6 total, 6 up, 6 in 2024-09-21T09:46:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:06 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:46:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:07 smithi033 ceph-mon[39045]: purged_snaps scrub starts 2024-09-21T09:46:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:07 smithi033 ceph-mon[39045]: purged_snaps scrub ok 2024-09-21T09:46:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:07 smithi033 ceph-mon[39045]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:46:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:07 smithi033 ceph-mon[27969]: purged_snaps scrub starts 2024-09-21T09:46:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:07 smithi033 ceph-mon[27969]: purged_snaps scrub ok 2024-09-21T09:46:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:07 smithi033 ceph-mon[27969]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:46:08.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:07 smithi105 ceph-mon[31286]: purged_snaps scrub starts 2024-09-21T09:46:08.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:07 smithi105 ceph-mon[31286]: purged_snaps scrub ok 2024-09-21T09:46:08.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:07 smithi105 ceph-mon[31286]: osdmap e37: 6 total, 6 up, 6 in 2024-09-21T09:46:08.855 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:08.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:46:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:08.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: osdmap e38: 6 total, 6 up, 6 in 2024-09-21T09:46:08.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:08.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:08.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:08.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:08.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:09 smithi105 ceph-mon[31286]: Detected new or changed devices on smithi105 2024-09-21T09:46:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:09 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi105 to 8544M 2024-09-21T09:46:09.824 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:46:09.844 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi105:vg_nvme/lv_2 2024-09-21T09:46:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:09 smithi033 ceph-mon[39045]: Detected new or changed devices on smithi105 2024-09-21T09:46:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:09 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi105 to 8544M 2024-09-21T09:46:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:09 smithi033 ceph-mon[27969]: Detected new or changed devices on smithi105 2024-09-21T09:46:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:09 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi105 to 8544M 2024-09-21T09:46:10.084 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:46:10.891 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:10 smithi105 ceph-mon[31286]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:10.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:10 smithi033 ceph-mon[27969]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:10.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:10 smithi033 ceph-mon[39045]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:12.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:12 smithi033 ceph-mon[39045]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:12.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:12.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:12.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:12 smithi033 ceph-mon[27969]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:12 smithi105 ceph-mon[31286]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:13.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:13 smithi105 ceph-mon[31286]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:13 smithi033 ceph-mon[27969]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:13 smithi033 ceph-mon[39045]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[27969]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/1645811777' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]: dispatch 2024-09-21T09:46:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]: dispatch 2024-09-21T09:46:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]': finished 2024-09-21T09:46:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[27969]: osdmap e39: 7 total, 6 up, 7 in 2024-09-21T09:46:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:14.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[39045]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:14.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/1645811777' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]: dispatch 2024-09-21T09:46:14.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]: dispatch 2024-09-21T09:46:14.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]': finished 2024-09-21T09:46:14.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[39045]: osdmap e39: 7 total, 6 up, 7 in 2024-09-21T09:46:14.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:14 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:14 smithi105 ceph-mon[31286]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:14 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/1645811777' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]: dispatch 2024-09-21T09:46:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:14 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]: dispatch 2024-09-21T09:46:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:14 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a57f3528-3cfb-4319-bf62-fa3a0762cfe4"}]': finished 2024-09-21T09:46:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:14 smithi105 ceph-mon[31286]: osdmap e39: 7 total, 6 up, 7 in 2024-09-21T09:46:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:14 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:15 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/2847043567' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:15 smithi033 ceph-mon[39045]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:15 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/2847043567' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:15 smithi033 ceph-mon[27969]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:15 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/2847043567' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:15 smithi105 ceph-mon[31286]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:18.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:18 smithi033 ceph-mon[27969]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:18 smithi033 ceph-mon[39045]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:18.773 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:18 smithi105 ceph-mon[31286]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:20.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:20 smithi033 ceph-mon[27969]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:20.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:20 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:46:20.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:20 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:20.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:20 smithi033 ceph-mon[39045]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:20.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:20 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:46:20.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:20 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:20.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:20 smithi105 ceph-mon[31286]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:20.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:20 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-21T09:46:20.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:20 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:21.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:21 smithi033 ceph-mon[27969]: Deploying daemon osd.6 on smithi105 2024-09-21T09:46:21.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:21 smithi033 ceph-mon[39045]: Deploying daemon osd.6 on smithi105 2024-09-21T09:46:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:21 smithi105 ceph-mon[31286]: Deploying daemon osd.6 on smithi105 2024-09-21T09:46:22.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:22 smithi033 ceph-mon[27969]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:22.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:22 smithi033 ceph-mon[39045]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:22.759 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:22 smithi105 ceph-mon[31286]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:24.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:24 smithi033 ceph-mon[27969]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:24.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:24 smithi033 ceph-mon[39045]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:24.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:24 smithi105 ceph-mon[31286]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:26 smithi105 ceph-mon[31286]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:26 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:26 smithi033 ceph-mon[39045]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:26 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:26 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:26 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:26.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:26 smithi033 ceph-mon[27969]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:26.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:26.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:26.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:26 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:28.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:28 smithi105 ceph-mon[31286]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:28.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:28 smithi033 ceph-mon[27969]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:28 smithi033 ceph-mon[39045]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:28.764 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 6 on host 'smithi105' 2024-09-21T09:46:29.381 DEBUG:teuthology.orchestra.run.smithi105:osd.6> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.6.service 2024-09-21T09:46:29.383 INFO:tasks.cephadm:Deploying osd.7 on smithi105 with /dev/vg_nvme/lv_1... 2024-09-21T09:46:29.383 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- lvm zap /dev/vg_nvme/lv_1 2024-09-21T09:46:29.679 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:46:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:29 smithi033 ceph-mon[39045]: from='osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:46:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:29 smithi033 ceph-mon[39045]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:46:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:29 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:29.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:29 smithi033 ceph-mon[27969]: from='osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:46:29.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:29 smithi033 ceph-mon[27969]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:46:29.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:29 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:29.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:29 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:29.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:29 smithi105 ceph-mon[31286]: from='osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:46:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:29 smithi105 ceph-mon[31286]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-21T09:46:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:29 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:29 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[39045]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[39045]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:46:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[39045]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:46:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[39045]: from='osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[39045]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:30.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[27969]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:30.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[27969]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:46:30.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[27969]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:46:30.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:30.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[27969]: from='osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:30.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[27969]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:30.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:30.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:30 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:30.755 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-mon[31286]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:30.755 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-mon[31286]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:46:30.755 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-mon[31286]: osdmap e40: 7 total, 6 up, 7 in 2024-09-21T09:46:30.755 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:30.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-mon[31286]: from='osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:30.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-mon[31286]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:30.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:30.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:30.756 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[49024]: 2024-09-21T09:46:30.454+0000 7f09420ba640 -1 osd.6 0 waiting for initial osdmap 2024-09-21T09:46:30.756 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:46:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[49024]: 2024-09-21T09:46:30.460+0000 7f093d6f2640 -1 osd.6 41 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:46:31.561 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:31 smithi105 ceph-mon[31286]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:31.562 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:31 smithi105 ceph-mon[31286]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:46:31.562 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:31 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:31.562 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:31 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:31.562 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:31 smithi105 ceph-mon[31286]: osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975] boot 2024-09-21T09:46:31.562 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:31 smithi105 ceph-mon[31286]: osdmap e42: 7 total, 7 up, 7 in 2024-09-21T09:46:31.562 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:31 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:31.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[39045]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:31.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[39045]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:46:31.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:31.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:31.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[39045]: osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975] boot 2024-09-21T09:46:31.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[39045]: osdmap e42: 7 total, 7 up, 7 in 2024-09-21T09:46:31.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:31.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[27969]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:31.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[27969]: osdmap e41: 7 total, 6 up, 7 in 2024-09-21T09:46:31.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:31.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:31.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[27969]: osd.6 [v2:172.21.15.105:6816/592734975,v1:172.21.15.105:6817/592734975] boot 2024-09-21T09:46:31.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[27969]: osdmap e42: 7 total, 7 up, 7 in 2024-09-21T09:46:31.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:31 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:46:32.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[27969]: purged_snaps scrub starts 2024-09-21T09:46:32.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[27969]: purged_snaps scrub ok 2024-09-21T09:46:32.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[27969]: pgmap v144: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:32.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[27969]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:46:32.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[27969]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:46:32.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[39045]: purged_snaps scrub starts 2024-09-21T09:46:32.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[39045]: purged_snaps scrub ok 2024-09-21T09:46:32.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[39045]: pgmap v144: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:32.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[39045]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:46:32.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:32 smithi033 ceph-mon[39045]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:46:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:32 smithi105 ceph-mon[31286]: purged_snaps scrub starts 2024-09-21T09:46:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:32 smithi105 ceph-mon[31286]: purged_snaps scrub ok 2024-09-21T09:46:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:32 smithi105 ceph-mon[31286]: pgmap v144: 1 pgs: 1 peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-21T09:46:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:32 smithi105 ceph-mon[31286]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:46:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:32 smithi105 ceph-mon[31286]: osdmap e43: 7 total, 7 up, 7 in 2024-09-21T09:46:32.983 INFO:teuthology.orchestra.run.smithi105.stdout: 2024-09-21T09:46:33.006 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch daemon add osd smithi105:vg_nvme/lv_1 2024-09-21T09:46:33.581 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:46:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:34 smithi033 ceph-mon[39045]: pgmap v147: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:34 smithi033 ceph-mon[39045]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:46:34.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:34 smithi033 ceph-mon[27969]: pgmap v147: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:34.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:34 smithi033 ceph-mon[27969]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:46:34.771 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:34 smithi105 ceph-mon[31286]: pgmap v147: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:34.771 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:34 smithi105 ceph-mon[31286]: osdmap e44: 7 total, 7 up, 7 in 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: Detected new or changed devices on smithi105 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi105 to 5696M 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:36.756 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:36.757 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.757 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:36.757 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:36.757 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:36.757 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.757 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:36.757 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:36.757 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:36 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: Detected new or changed devices on smithi105 2024-09-21T09:46:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi105 to 5696M 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: pgmap v149: 1 pgs: 1 peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: Detected new or changed devices on smithi105 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi105 to 5696M 2024-09-21T09:46:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-21T09:46:36.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:36 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:37 smithi105 ceph-mon[31286]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:37 smithi033 ceph-mon[27969]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:37 smithi033 ceph-mon[39045]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi105:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:46:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[27969]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:46:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[27969]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:46:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[27969]: Cluster is now healthy 2024-09-21T09:46:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/1653511305' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]: dispatch 2024-09-21T09:46:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]: dispatch 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[27969]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]': finished 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[27969]: osdmap e45: 8 total, 7 up, 8 in 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[39045]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[39045]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[39045]: Cluster is now healthy 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/1653511305' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]: dispatch 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]: dispatch 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[39045]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]': finished 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[39045]: osdmap e45: 8 total, 7 up, 8 in 2024-09-21T09:46:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:38 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:39.017 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:38 smithi105 ceph-mon[31286]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-21T09:46:39.017 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:38 smithi105 ceph-mon[31286]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:46:39.018 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:38 smithi105 ceph-mon[31286]: Cluster is now healthy 2024-09-21T09:46:39.018 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:38 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/1653511305' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]: dispatch 2024-09-21T09:46:39.018 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:38 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]: dispatch 2024-09-21T09:46:39.018 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:38 smithi105 ceph-mon[31286]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cbbae365-68d8-4e30-bf12-b1049489cb50"}]': finished 2024-09-21T09:46:39.018 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:38 smithi105 ceph-mon[31286]: osdmap e45: 8 total, 7 up, 8 in 2024-09-21T09:46:39.018 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:38 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:39 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/922912913' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:39 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/922912913' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:39 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/922912913' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-21T09:46:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:40 smithi033 ceph-mon[39045]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:46:40.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:40 smithi033 ceph-mon[27969]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:46:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:40 smithi105 ceph-mon[31286]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-21T09:46:42.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:42 smithi033 ceph-mon[27969]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:46:42.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:42 smithi033 ceph-mon[39045]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:46:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:42 smithi105 ceph-mon[31286]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-21T09:46:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:44 smithi105 ceph-mon[31286]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:46:44.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:44 smithi033 ceph-mon[27969]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:46:44.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:44 smithi033 ceph-mon[39045]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:46:45.932 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:45 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:46:45.932 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:45 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:46:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:45 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-21T09:46:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:45 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:46.769 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:46 smithi105 ceph-mon[31286]: Deploying daemon osd.7 on smithi105 2024-09-21T09:46:46.769 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:46 smithi105 ceph-mon[31286]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:46:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:46 smithi033 ceph-mon[27969]: Deploying daemon osd.7 on smithi105 2024-09-21T09:46:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:46 smithi033 ceph-mon[27969]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:46:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:46 smithi033 ceph-mon[39045]: Deploying daemon osd.7 on smithi105 2024-09-21T09:46:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:46 smithi033 ceph-mon[39045]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-21T09:46:48.955 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:48 smithi105 ceph-mon[31286]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:48.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:48 smithi033 ceph-mon[27969]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:48.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:48 smithi033 ceph-mon[39045]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:50 smithi033 ceph-mon[27969]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:50 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:50 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:50 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:50 smithi033 ceph-mon[39045]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:50 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:50 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:50 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:50 smithi105 ceph-mon[31286]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:50 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:50 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:50.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:50 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:52.804 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:52 smithi105 ceph-mon[31286]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:52 smithi033 ceph-mon[27969]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:52 smithi033 ceph-mon[39045]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:53.458 INFO:teuthology.orchestra.run.smithi105.stdout:Created osd(s) 7 on host 'smithi105' 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:53 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.149 DEBUG:teuthology.orchestra.run.smithi105:osd.7> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.7.service 2024-09-21T09:46:54.151 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-21T09:46:54.151 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd stat -f json 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.180 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.181 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:53 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:54.418 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[27969]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:46:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[27969]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:46:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[27969]: from='osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[27969]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[39045]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:46:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[39045]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:46:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[39045]: from='osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:54 smithi033 ceph-mon[39045]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:55.253 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:54 smithi105 ceph-mon[31286]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:46:55.253 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:54 smithi105 ceph-mon[31286]: osdmap e46: 8 total, 7 up, 8 in 2024-09-21T09:46:55.253 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:54 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:55.253 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:54 smithi105 ceph-mon[31286]: from='osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:55.253 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:54 smithi105 ceph-mon[31286]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]: dispatch 2024-09-21T09:46:55.254 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:46:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[55164]: 2024-09-21T09:46:54.799+0000 7f5967787640 -1 osd.7 0 waiting for initial osdmap 2024-09-21T09:46:55.254 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:46:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[55164]: 2024-09-21T09:46:54.806+0000 7f5960598640 -1 osd.7 47 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:46:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:55 smithi105 ceph-mon[31286]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:55 smithi105 ceph-mon[31286]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:46:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:55 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:55 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:55 smithi105 ceph-mon[31286]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:55 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.071 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[39045]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[39045]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[39045]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[27969]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]}]': finished 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[27969]: osdmap e47: 8 total, 7 up, 8 in 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.072 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[27969]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-21T09:46:56.073 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:55 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.454 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:46:56.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:56 smithi105 ceph-mon[31286]: purged_snaps scrub starts 2024-09-21T09:46:56.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:56 smithi105 ceph-mon[31286]: purged_snaps scrub ok 2024-09-21T09:46:56.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:56 smithi105 ceph-mon[31286]: osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701] boot 2024-09-21T09:46:56.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:56 smithi105 ceph-mon[31286]: osdmap e48: 8 total, 8 up, 8 in 2024-09-21T09:46:56.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:56 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:56 smithi105 ceph-mon[31286]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:46:56.894 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:56 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1701075063' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[39045]: purged_snaps scrub starts 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[39045]: purged_snaps scrub ok 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[39045]: osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701] boot 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[39045]: osdmap e48: 8 total, 8 up, 8 in 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[39045]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1701075063' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[27969]: purged_snaps scrub starts 2024-09-21T09:46:56.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[27969]: purged_snaps scrub ok 2024-09-21T09:46:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[27969]: osd.7 [v2:172.21.15.105:6824/455831701,v1:172.21.15.105:6825/455831701] boot 2024-09-21T09:46:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[27969]: osdmap e48: 8 total, 8 up, 8 in 2024-09-21T09:46:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:46:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[27969]: osdmap e49: 8 total, 8 up, 8 in 2024-09-21T09:46:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:56 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1701075063' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-21T09:46:57.167 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":49,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912015,"num_in_osds":8,"osd_in_since":1726911998,"num_remapped_pgs":0} 2024-09-21T09:46:57.168 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:46:57.463 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: Detected new or changed devices on smithi105 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi105 to 4272M 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: pgmap v165: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:58.183 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:57 smithi105 ceph-mon[31286]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:46:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: Detected new or changed devices on smithi105 2024-09-21T09:46:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi105 to 4272M 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: pgmap v165: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[39045]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: Detected new or changed devices on smithi105 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi105 to 4272M 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: pgmap v165: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:46:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:57 smithi033 ceph-mon[27969]: osdmap e50: 8 total, 8 up, 8 in 2024-09-21T09:46:59.253 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:46:59.253 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":50,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T09:46:57.407162+0000","last_up_change":"2024-09-21T09:46:55.803410+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6803","nonce":3277300496}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6805","nonce":3277300496}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6809","nonce":3277300496}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6807","nonce":3277300496}]},"public_addr":"172.21.15.33:6803/3277300496","cluster_addr":"172.21.15.33:6805/3277300496","heartbeat_back_addr":"172.21.15.33:6809/3277300496","heartbeat_front_addr":"172.21.15.33:6807/3277300496","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6811","nonce":2439650417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6813","nonce":2439650417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6817","nonce":2439650417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6815","nonce":2439650417}]},"public_addr":"172.21.15.33:6811/2439650417","cluster_addr":"172.21.15.33:6813/2439650417","heartbeat_back_addr":"172.21.15.33:6817/2439650417","heartbeat_front_addr":"172.21.15.33:6815/2439650417","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6819","nonce":3305661464}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6821","nonce":3305661464}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6825","nonce":3305661464}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6823","nonce":3305661464}]},"public_addr":"172.21.15.33:6819/3305661464","cluster_addr":"172.21.15.33:6821/3305661464","heartbeat_back_addr":"172.21.15.33:6825/3305661464","heartbeat_front_addr":"172.21.15.33:6823/3305661464","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6827","nonce":510343171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6829","nonce":510343171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6833","nonce":510343171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6831","nonce":510343171}]},"public_addr":"172.21.15.33:6827/510343171","cluster_addr":"172.21.15.33:6829/510343171","heartbeat_back_addr":"172.21.15.33:6833/510343171","heartbeat_front_addr":"172.21.15.33:6831/510343171","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6801","nonce":1234622360}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6803","nonce":1234622360}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6807","nonce":1234622360}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6805","nonce":1234622360}]},"public_addr":"172.21.15.105:6801/1234622360","cluster_addr":"172.21.15.105:6803/1234622360","heartbeat_back_addr":"172.21.15.105:6807/1234622360","heartbeat_front_addr":"172.21.15.105:6805/1234622360","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6809","nonce":1890006054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6811","nonce":1890006054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6815","nonce":1890006054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6813","nonce":1890006054}]},"public_addr":"172.21.15.105:6809/1890006054","cluster_addr":"172.21.15.105:6811/1890006054","heartbeat_back_addr":"172.21.15.105:6815/1890006054","heartbeat_front_addr":"172.21.15.105:6813/1890006054","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6817","nonce":592734975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6819","nonce":592734975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6823","nonce":592734975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6821","nonce":592734975}]},"public_addr":"172.21.15.105:6817/592734975","cluster_addr":"172.21.15.105:6819/592734975","heartbeat_back_addr":"172.21.15.105:6823/592734975","heartbeat_front_addr":"172.21.15.105:6821/592734975","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6825","nonce":455831701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6827","nonce":455831701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6831","nonce":455831701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6829","nonce":455831701}]},"public_addr":"172.21.15.105:6825/455831701","cluster_addr":"172.21.15.105:6827/455831701","heartbeat_back_addr":"172.21.15.105:6831/455831701","heartbeat_front_addr":"172.21.15.105:6829/455831701","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:46:59.451 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:46:59 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1872780150' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:46:59.614 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:46:59 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1872780150' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:46:59.614 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:46:59 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1872780150' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:46:59.963 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-21T09:44:57.819538+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-21T09:46:59.963 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool get .mgr pg_num 2024-09-21T09:47:00.225 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:00.601 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:00 smithi033 ceph-mon[39045]: pgmap v167: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:00.601 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:00 smithi033 ceph-mon[27969]: pgmap v167: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:00 smithi105 ceph-mon[31286]: pgmap v167: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:02.149 INFO:teuthology.orchestra.run.smithi033.stdout:pg_num: 1 2024-09-21T09:47:02.737 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:02 smithi033 ceph-mon[27969]: pgmap v168: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:02.737 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:02 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/434258803' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:47:02.737 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:02 smithi033 ceph-mon[39045]: pgmap v168: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:02.737 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:02 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/434258803' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:47:02.764 INFO:tasks.cephadm:Adding mds.a on smithi033 2024-09-21T09:47:02.764 INFO:tasks.cephadm:Adding mds.b on smithi105 2024-09-21T09:47:02.765 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch apply mds all '2;smithi033=a;smithi105=b' 2024-09-21T09:47:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:02 smithi105 ceph-mon[31286]: pgmap v168: 1 pgs: 1 peering; 0 B data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:02 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/434258803' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-21T09:47:03.023 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:47:04.686 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:04 smithi105 ceph-mon[31286]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:04.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:04 smithi033 ceph-mon[39045]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:04.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:04 smithi033 ceph-mon[27969]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:04.848 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled mds.all update... 2024-09-21T09:47:05.422 DEBUG:teuthology.orchestra.run.smithi033:mds.a> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.a.service 2024-09-21T09:47:05.424 DEBUG:teuthology.orchestra.run.smithi105:mds.b> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.b.service 2024-09-21T09:47:05.464 INFO:tasks.cephadm:Setting up CephFS filesystem(s)... 2024-09-21T09:47:05.464 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:47:06.244 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi033=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: Saving service mds.all spec with placement smithi033=a;smithi105=b;count:2 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: Deploying daemon mds.a on smithi033 2024-09-21T09:47:06.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[39045]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi033=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: Saving service mds.all spec with placement smithi033=a;smithi105=b;count:2 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: Deploying daemon mds.a on smithi033 2024-09-21T09:47:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:05 smithi033 ceph-mon[27969]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='client.24343 -' entity='client.admin' cmd=[{"prefix": "orch apply mds", "fs_name": "all", "placement": "2;smithi033=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: Saving service mds.all spec with placement smithi033=a;smithi105=b;count:2 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: Deploying daemon mds.a on smithi033 2024-09-21T09:47:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:05 smithi105 ceph-mon[31286]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:08.533 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:08.533 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":1,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[],"filesystems":[]} 2024-09-21T09:47:08.534 INFO:teuthology.orchestra.run.smithi033.stderr:dumped fsmap epoch 1 2024-09-21T09:47:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:08 smithi033 ceph-mon[27969]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:08 smithi033 ceph-mon[39045]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:08 smithi105 ceph-mon[31286]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:09.430 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:47:09 smithi033 systemd[1]: Started Ceph mds.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:47:09.739 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/535207299' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:09.739 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: mds.? [v2:172.21.15.33:6834/125068843,v1:172.21.15.33:6835/125068843] up:boot 2024-09-21T09:47:09.739 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: fsmap 1 up:standby 2024-09-21T09:47:09.739 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:47:09.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:47:09.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:47:09.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/535207299' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: mds.? [v2:172.21.15.33:6834/125068843,v1:172.21.15.33:6835/125068843] up:boot 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: fsmap 1 up:standby 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:47:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/535207299' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: mds.? [v2:172.21.15.33:6834/125068843,v1:172.21.15.33:6835/125068843] up:boot 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: fsmap 1 up:standby 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-21T09:47:09.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:10.738 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:10 smithi105 ceph-mon[31286]: Deploying daemon mds.b on smithi105 2024-09-21T09:47:10.738 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:10 smithi105 ceph-mon[31286]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:10.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:10 smithi033 ceph-mon[27969]: Deploying daemon mds.b on smithi105 2024-09-21T09:47:10.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:10 smithi033 ceph-mon[27969]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:10 smithi033 ceph-mon[39045]: Deploying daemon mds.b on smithi105 2024-09-21T09:47:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:10 smithi033 ceph-mon[39045]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:11.147 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:47:11.416 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:11.520 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 09:47:11 smithi105 podman[59270]: 2024-09-21 09:47:11.363430104 +0000 UTC m=+0.137921658 container create b0497afc1e278413e71cd22556581c898dac9b7e86c9592845aa8e4a3eddca3e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True) 2024-09-21T09:47:12.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 09:47:11 smithi105 podman[59270]: 2024-09-21 09:47:11.640897354 +0000 UTC m=+0.415388909 container init b0497afc1e278413e71cd22556581c898dac9b7e86c9592845aa8e4a3eddca3e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:47:12.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 09:47:11 smithi105 podman[59270]: 2024-09-21 09:47:11.645789447 +0000 UTC m=+0.420281005 container start b0497afc1e278413e71cd22556581c898dac9b7e86c9592845aa8e4a3eddca3e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T09:47:12.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 09:47:11 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b[59286]: starting mds.b at 2024-09-21T09:47:12.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 09:47:11 smithi105 bash[59270]: b0497afc1e278413e71cd22556581c898dac9b7e86c9592845aa8e4a3eddca3e 2024-09-21T09:47:12.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 09:47:11 smithi105 systemd[1]: Started Ceph mds.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[39045]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[27969]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:12.735 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.736 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:12 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:12.762 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:12 smithi105 ceph-mon[31286]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:12.762 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.762 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.763 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.763 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.763 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:12.763 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:12 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:13.152 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:13.152 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":3,"default_fscid":-1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24323,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/125068843","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":125068843},{"type":"v1","addr":"172.21.15.33:6835","nonce":125068843}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2},{"gid":24358,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2691139879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2691139879},{"type":"v1","addr":"172.21.15.105:6833","nonce":2691139879}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":3}],"filesystems":[]} 2024-09-21T09:47:13.152 INFO:teuthology.orchestra.run.smithi033.stderr:dumped fsmap epoch 3 2024-09-21T09:47:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[39045]: mds.? [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] up:boot 2024-09-21T09:47:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[39045]: fsmap 2 up:standby 2024-09-21T09:47:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:47:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3706238467' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:13.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[27969]: mds.? [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] up:boot 2024-09-21T09:47:13.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[27969]: fsmap 2 up:standby 2024-09-21T09:47:13.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:47:13.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3706238467' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:13.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:13.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:13 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:13 smithi105 ceph-mon[31286]: mds.? [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] up:boot 2024-09-21T09:47:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:13 smithi105 ceph-mon[31286]: fsmap 2 up:standby 2024-09-21T09:47:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:13 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:47:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:13 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3706238467' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:13 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:13 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:14.565 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph --format=json-pretty osd lspools 2024-09-21T09:47:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:14 smithi033 ceph-mon[27969]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:14 smithi033 ceph-mon[39045]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:14 smithi105 ceph-mon[31286]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:14.855 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:15.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:15 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:47:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:47:16.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:47:16.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:16.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:16.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:15 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:47:16.747 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:16.747 INFO:teuthology.orchestra.run.smithi033.stdout:[ 2024-09-21T09:47:16.747 INFO:teuthology.orchestra.run.smithi033.stdout: { 2024-09-21T09:47:16.747 INFO:teuthology.orchestra.run.smithi033.stdout: "poolnum": 1, 2024-09-21T09:47:16.747 INFO:teuthology.orchestra.run.smithi033.stdout: "poolname": ".mgr" 2024-09-21T09:47:16.747 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:47:16.747 INFO:teuthology.orchestra.run.smithi033.stdout:] 2024-09-21T09:47:17.056 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:16 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1030338979' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-21T09:47:17.057 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:16 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1030338979' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-21T09:47:17.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:16 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1030338979' entity='client.admin' cmd=[{"prefix": "osd lspools", "format": "json-pretty"}]: dispatch 2024-09-21T09:47:17.446 DEBUG:tasks.cephfs.filesystem:Creating filesystem 'cephfs' 2024-09-21T09:47:17.447 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create cephfs_metadata --pg_num_min 64 2024-09-21T09:47:17.706 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:18.111 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:17 smithi033 ceph-mon[39045]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:18.111 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:17 smithi033 ceph-mon[27969]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:17 smithi105 ceph-mon[31286]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:19.855 INFO:teuthology.orchestra.run.smithi033.stderr:pool 'cephfs_metadata' created 2024-09-21T09:47:20.154 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:19 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1172283475' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:47:20.155 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:19 smithi033 ceph-mon[39045]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:20.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:19 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1172283475' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:47:20.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:19 smithi033 ceph-mon[27969]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:20.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:19 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1172283475' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]: dispatch 2024-09-21T09:47:20.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:19 smithi105 ceph-mon[31286]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:20.466 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create cephfs_data 64 --pg_num_min 64 --target_size_ratio 0.9 2024-09-21T09:47:20.725 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:21.107 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:20 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1172283475' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-21T09:47:21.107 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:20 smithi033 ceph-mon[39045]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:47:21.107 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:20 smithi033 ceph-mon[39045]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:47:21.108 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:20 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1172283475' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-21T09:47:21.108 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:20 smithi033 ceph-mon[27969]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:47:21.108 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:20 smithi033 ceph-mon[27969]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:47:21.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:20 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1172283475' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_metadata", "pg_num_min": 64}]': finished 2024-09-21T09:47:21.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:20 smithi105 ceph-mon[31286]: osdmap e51: 8 total, 8 up, 8 in 2024-09-21T09:47:21.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:20 smithi105 ceph-mon[31286]: osdmap e52: 8 total, 8 up, 8 in 2024-09-21T09:47:22.150 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:21 smithi033 ceph-mon[39045]: pgmap v180: 33 pgs: 3 creating+activating, 2 creating+peering, 27 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:22.151 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:21 smithi033 ceph-mon[27969]: pgmap v180: 33 pgs: 3 creating+activating, 2 creating+peering, 27 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:21 smithi105 ceph-mon[31286]: pgmap v180: 33 pgs: 3 creating+activating, 2 creating+peering, 27 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:23.033 INFO:teuthology.orchestra.run.smithi033.stderr:pool 'cephfs_data' created 2024-09-21T09:47:23.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:22 smithi033 ceph-mon[39045]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:23.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:22 smithi033 ceph-mon[39045]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:47:23.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:22 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2901875192' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-21T09:47:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:22 smithi033 ceph-mon[27969]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:22 smithi033 ceph-mon[27969]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:47:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:22 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2901875192' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-21T09:47:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:22 smithi105 ceph-mon[31286]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:47:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:22 smithi105 ceph-mon[31286]: osdmap e53: 8 total, 8 up, 8 in 2024-09-21T09:47:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:22 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2901875192' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]: dispatch 2024-09-21T09:47:23.750 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs new cephfs cephfs_metadata cephfs_data 2024-09-21T09:47:24.012 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:24.390 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:24 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2901875192' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-21T09:47:24.390 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:24 smithi033 ceph-mon[39045]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:47:24.390 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:24 smithi033 ceph-mon[39045]: pgmap v183: 97 pgs: 22 active+clean, 3 creating+activating, 2 creating+peering, 70 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:24.390 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:24 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2901875192' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-21T09:47:24.390 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:24 smithi033 ceph-mon[27969]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:47:24.391 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:24 smithi033 ceph-mon[27969]: pgmap v183: 97 pgs: 22 active+clean, 3 creating+activating, 2 creating+peering, 70 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:24 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2901875192' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "cephfs_data", "pg_num": 64, "pg_num_min": 64, "target_size_ratio": 0.9}]': finished 2024-09-21T09:47:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:24 smithi105 ceph-mon[31286]: osdmap e54: 8 total, 8 up, 8 in 2024-09-21T09:47:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:24 smithi105 ceph-mon[31286]: pgmap v183: 97 pgs: 22 active+clean, 3 creating+activating, 2 creating+peering, 70 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:25.244 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:25 smithi033 ceph-mon[39045]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:47:25.245 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:25 smithi033 ceph-mon[27969]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:47:25.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:25 smithi105 ceph-mon[31286]: osdmap e55: 8 total, 8 up, 8 in 2024-09-21T09:47:26.051 INFO:teuthology.orchestra.run.smithi033.stderr:new fs with metadata pool 2 and data pool 3 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[39045]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[39045]: pgmap v186: 97 pgs: 34 active+clean, 3 creating+activating, 2 creating+peering, 58 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3056369754' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[39045]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[27969]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[27969]: pgmap v186: 97 pgs: 34 active+clean, 3 creating+activating, 2 creating+peering, 58 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3056369754' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:47:26.427 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:26 smithi033 ceph-mon[27969]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-21T09:47:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:26 smithi105 ceph-mon[31286]: osdmap e56: 8 total, 8 up, 8 in 2024-09-21T09:47:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:26 smithi105 ceph-mon[31286]: pgmap v186: 97 pgs: 34 active+clean, 3 creating+activating, 2 creating+peering, 58 unknown; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:26 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3056369754' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:47:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:26 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]: dispatch 2024-09-21T09:47:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:26 smithi105 ceph-mon[31286]: daemon mds.b assigned to filesystem cephfs as rank 0 2024-09-21T09:47:26.839 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:47:27.096 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[39045]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[39045]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[39045]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[39045]: mds.? [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] up:active 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[39045]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[27969]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[27969]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-21T09:47:27.195 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[27969]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-21T09:47:27.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[27969]: mds.? [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] up:active 2024-09-21T09:47:27.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:27 smithi033 ceph-mon[27969]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:27.446 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:27 smithi105 ceph-mon[31286]: osdmap e57: 8 total, 8 up, 8 in 2024-09-21T09:47:27.446 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:27 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs new", "fs_name": "cephfs", "metadata": "cephfs_metadata", "data": "cephfs_data"}]': finished 2024-09-21T09:47:27.446 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:27 smithi105 ceph-mon[31286]: fsmap cephfs:1 {0=b=up:creating} 1 up:standby 2024-09-21T09:47:27.446 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:27 smithi105 ceph-mon[31286]: daemon mds.b is now active in filesystem cephfs as rank 0 2024-09-21T09:47:27.446 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:27 smithi105 ceph-mon[31286]: mds.? [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] up:active 2024-09-21T09:47:27.446 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:27 smithi105 ceph-mon[31286]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:27.780 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 09:47:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b[59286]: 2024-09-21T09:47:27.443+0000 7fe236fa1640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-21T09:47:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:28 smithi033 ceph-mon[27969]: pgmap v188: 97 pgs: 97 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 941 B/s wr, 7 op/s 2024-09-21T09:47:28.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:28 smithi033 ceph-mon[39045]: pgmap v188: 97 pgs: 97 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 941 B/s wr, 7 op/s 2024-09-21T09:47:28.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:28 smithi105 ceph-mon[31286]: pgmap v188: 97 pgs: 97 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 941 B/s wr, 7 op/s 2024-09-21T09:47:28.993 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:28.993 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":57,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T09:47:25.959311+0000","last_up_change":"2024-09-21T09:46:55.803410+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:47:19.790061+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:47:22.976967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6803","nonce":3277300496}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6805","nonce":3277300496}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6809","nonce":3277300496}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6807","nonce":3277300496}]},"public_addr":"172.21.15.33:6803/3277300496","cluster_addr":"172.21.15.33:6805/3277300496","heartbeat_back_addr":"172.21.15.33:6809/3277300496","heartbeat_front_addr":"172.21.15.33:6807/3277300496","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6811","nonce":2439650417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6813","nonce":2439650417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6817","nonce":2439650417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6815","nonce":2439650417}]},"public_addr":"172.21.15.33:6811/2439650417","cluster_addr":"172.21.15.33:6813/2439650417","heartbeat_back_addr":"172.21.15.33:6817/2439650417","heartbeat_front_addr":"172.21.15.33:6815/2439650417","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6819","nonce":3305661464}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6821","nonce":3305661464}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6825","nonce":3305661464}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6823","nonce":3305661464}]},"public_addr":"172.21.15.33:6819/3305661464","cluster_addr":"172.21.15.33:6821/3305661464","heartbeat_back_addr":"172.21.15.33:6825/3305661464","heartbeat_front_addr":"172.21.15.33:6823/3305661464","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6827","nonce":510343171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6829","nonce":510343171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6833","nonce":510343171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6831","nonce":510343171}]},"public_addr":"172.21.15.33:6827/510343171","cluster_addr":"172.21.15.33:6829/510343171","heartbeat_back_addr":"172.21.15.33:6833/510343171","heartbeat_front_addr":"172.21.15.33:6831/510343171","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6801","nonce":1234622360}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6803","nonce":1234622360}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6807","nonce":1234622360}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6805","nonce":1234622360}]},"public_addr":"172.21.15.105:6801/1234622360","cluster_addr":"172.21.15.105:6803/1234622360","heartbeat_back_addr":"172.21.15.105:6807/1234622360","heartbeat_front_addr":"172.21.15.105:6805/1234622360","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6809","nonce":1890006054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6811","nonce":1890006054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6815","nonce":1890006054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6813","nonce":1890006054}]},"public_addr":"172.21.15.105:6809/1890006054","cluster_addr":"172.21.15.105:6811/1890006054","heartbeat_back_addr":"172.21.15.105:6815/1890006054","heartbeat_front_addr":"172.21.15.105:6813/1890006054","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6817","nonce":592734975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6819","nonce":592734975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6823","nonce":592734975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6821","nonce":592734975}]},"public_addr":"172.21.15.105:6817/592734975","cluster_addr":"172.21.15.105:6819/592734975","heartbeat_back_addr":"172.21.15.105:6823/592734975","heartbeat_front_addr":"172.21.15.105:6821/592734975","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6825","nonce":455831701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6827","nonce":455831701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6831","nonce":455831701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6829","nonce":455831701}]},"public_addr":"172.21.15.105:6825/455831701","cluster_addr":"172.21.15.105:6827/455831701","heartbeat_back_addr":"172.21.15.105:6831/455831701","heartbeat_front_addr":"172.21.15.105:6829/455831701","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:47:29.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:29 smithi033 ceph-mon[27969]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-21T09:47:29.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:29 smithi033 ceph-mon[27969]: Cluster is now healthy 2024-09-21T09:47:29.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:29 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2729511353' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:29 smithi033 ceph-mon[39045]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-21T09:47:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:29 smithi033 ceph-mon[39045]: Cluster is now healthy 2024-09-21T09:47:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:29 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2729511353' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:29.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:29 smithi105 ceph-mon[31286]: Health check cleared: POOL_APP_NOT_ENABLED (was: 2 pool(s) do not have an application enabled) 2024-09-21T09:47:29.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:29 smithi105 ceph-mon[31286]: Cluster is now healthy 2024-09-21T09:47:29.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:29 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2729511353' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:29.615 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:47:29.866 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:30.240 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:30 smithi033 ceph-mon[39045]: pgmap v189: 97 pgs: 97 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 B/s wr, 5 op/s 2024-09-21T09:47:30.241 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:30 smithi033 ceph-mon[27969]: pgmap v189: 97 pgs: 97 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 B/s wr, 5 op/s 2024-09-21T09:47:30.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:30 smithi105 ceph-mon[31286]: pgmap v189: 97 pgs: 97 active+clean; 578 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 682 B/s wr, 5 op/s 2024-09-21T09:47:31.645 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:31.645 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":57,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T09:47:25.959311+0000","last_up_change":"2024-09-21T09:46:55.803410+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:47:19.790061+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:47:22.976967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6803","nonce":3277300496}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6805","nonce":3277300496}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6809","nonce":3277300496}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6807","nonce":3277300496}]},"public_addr":"172.21.15.33:6803/3277300496","cluster_addr":"172.21.15.33:6805/3277300496","heartbeat_back_addr":"172.21.15.33:6809/3277300496","heartbeat_front_addr":"172.21.15.33:6807/3277300496","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6811","nonce":2439650417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6813","nonce":2439650417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6817","nonce":2439650417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6815","nonce":2439650417}]},"public_addr":"172.21.15.33:6811/2439650417","cluster_addr":"172.21.15.33:6813/2439650417","heartbeat_back_addr":"172.21.15.33:6817/2439650417","heartbeat_front_addr":"172.21.15.33:6815/2439650417","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6819","nonce":3305661464}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6821","nonce":3305661464}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6825","nonce":3305661464}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6823","nonce":3305661464}]},"public_addr":"172.21.15.33:6819/3305661464","cluster_addr":"172.21.15.33:6821/3305661464","heartbeat_back_addr":"172.21.15.33:6825/3305661464","heartbeat_front_addr":"172.21.15.33:6823/3305661464","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6827","nonce":510343171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6829","nonce":510343171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6833","nonce":510343171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6831","nonce":510343171}]},"public_addr":"172.21.15.33:6827/510343171","cluster_addr":"172.21.15.33:6829/510343171","heartbeat_back_addr":"172.21.15.33:6833/510343171","heartbeat_front_addr":"172.21.15.33:6831/510343171","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6801","nonce":1234622360}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6803","nonce":1234622360}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6807","nonce":1234622360}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6805","nonce":1234622360}]},"public_addr":"172.21.15.105:6801/1234622360","cluster_addr":"172.21.15.105:6803/1234622360","heartbeat_back_addr":"172.21.15.105:6807/1234622360","heartbeat_front_addr":"172.21.15.105:6805/1234622360","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6809","nonce":1890006054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6811","nonce":1890006054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6815","nonce":1890006054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6813","nonce":1890006054}]},"public_addr":"172.21.15.105:6809/1890006054","cluster_addr":"172.21.15.105:6811/1890006054","heartbeat_back_addr":"172.21.15.105:6815/1890006054","heartbeat_front_addr":"172.21.15.105:6813/1890006054","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6817","nonce":592734975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6819","nonce":592734975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6823","nonce":592734975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6821","nonce":592734975}]},"public_addr":"172.21.15.105:6817/592734975","cluster_addr":"172.21.15.105:6819/592734975","heartbeat_back_addr":"172.21.15.105:6823/592734975","heartbeat_front_addr":"172.21.15.105:6821/592734975","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6825","nonce":455831701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6827","nonce":455831701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6831","nonce":455831701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6829","nonce":455831701}]},"public_addr":"172.21.15.105:6825/455831701","cluster_addr":"172.21.15.105:6827/455831701","heartbeat_back_addr":"172.21.15.105:6831/455831701","heartbeat_front_addr":"172.21.15.105:6829/455831701","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:47:32.348 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs set cephfs standby_count_wanted 0 2024-09-21T09:47:32.604 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:32.725 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:32 smithi033 ceph-mon[39045]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s wr, 6 op/s 2024-09-21T09:47:32.725 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:32 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1687877643' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:32 smithi033 ceph-mon[27969]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s wr, 6 op/s 2024-09-21T09:47:32.725 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:32 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1687877643' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:32 smithi105 ceph-mon[31286]: pgmap v190: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.8 KiB/s wr, 6 op/s 2024-09-21T09:47:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:32 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1687877643' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:34 smithi033 ceph-mon[39045]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-21T09:47:34.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:34 smithi033 ceph-mon[27969]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-21T09:47:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:34 smithi105 ceph-mon[31286]: pgmap v191: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2024-09-21T09:47:35.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:35 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3214519914' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:47:35.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:35 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:47:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:35 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3214519914' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:47:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:35 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:47:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:35 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3214519914' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:47:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:35 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]: dispatch 2024-09-21T09:47:36.216 DEBUG:tasks.cephfs.filesystem:fs_config: {} 2024-09-21T09:47:36.216 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs set cephfs allow_standby_replay false 2024-09-21T09:47:36.478 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:36.712 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:36 smithi033 ceph-mon[39045]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:47:36.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:36 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-21T09:47:36.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:36 smithi033 ceph-mon[39045]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:36.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:36 smithi033 ceph-mon[27969]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:47:36.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:36 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-21T09:47:36.713 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:36 smithi033 ceph-mon[27969]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:36 smithi105 ceph-mon[31286]: pgmap v192: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2024-09-21T09:47:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:36 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "standby_count_wanted", "val": "0"}]': finished 2024-09-21T09:47:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:36 smithi105 ceph-mon[31286]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:38.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:38 smithi033 ceph-mon[27969]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:47:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:38 smithi033 ceph-mon[39045]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:47:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:38 smithi105 ceph-mon[31286]: pgmap v193: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-09-21T09:47:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:39 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3515449019' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-21T09:47:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:39 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3515449019' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-21T09:47:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:39 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3515449019' entity='client.admin' cmd=[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]: dispatch 2024-09-21T09:47:40.173 DEBUG:tasks.cephfs.filesystem:Not Creating any subvols for filesystem 'cephfs' 2024-09-21T09:47:40.173 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:47:40.435 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:40.457 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:40 smithi033 ceph-mon[27969]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-09-21T09:47:40.457 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:40 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3515449019' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-21T09:47:40.457 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:40 smithi033 ceph-mon[27969]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:40.708 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:40 smithi033 ceph-mon[39045]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-09-21T09:47:40.708 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:40 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3515449019' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-21T09:47:40.708 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:40 smithi033 ceph-mon[39045]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:40 smithi105 ceph-mon[31286]: pgmap v194: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-09-21T09:47:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:40 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3515449019' entity='client.admin' cmd='[{"prefix": "fs set", "fs_name": "cephfs", "var": "allow_standby_replay", "val": "false"}]': finished 2024-09-21T09:47:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:40 smithi105 ceph-mon[31286]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:47:42.353 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:42.353 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24323,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/125068843","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":125068843},{"type":"v1","addr":"172.21.15.33:6835","nonce":125068843}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:47:25.959476+0000","modified":"2024-09-21T09:47:39.447745+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24358},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24358":{"gid":24358,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.105:6833/2691139879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2691139879},{"type":"v1","addr":"172.21.15.105:6833","nonce":2691139879}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-21T09:47:42.354 INFO:teuthology.orchestra.run.smithi033.stderr:dumped fsmap epoch 7 2024-09-21T09:47:42.525 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:42 smithi033 ceph-mon[39045]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-09-21T09:47:42.525 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:42 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/987240071' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:42.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:42 smithi033 ceph-mon[27969]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-09-21T09:47:42.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:42 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/987240071' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:42 smithi105 ceph-mon[31286]: pgmap v195: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 767 B/s wr, 1 op/s 2024-09-21T09:47:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:42 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/987240071' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:47:43.239 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:47:43.500 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:44.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:44 smithi033 ceph-mon[39045]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:44.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:44 smithi033 ceph-mon[27969]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:44 smithi105 ceph-mon[31286]: pgmap v196: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:45.078 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:45.079 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":57,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T09:47:25.959311+0000","last_up_change":"2024-09-21T09:46:55.803410+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":3,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:47:19.790061+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:47:22.976967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6803","nonce":3277300496}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6805","nonce":3277300496}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6809","nonce":3277300496}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6807","nonce":3277300496}]},"public_addr":"172.21.15.33:6803/3277300496","cluster_addr":"172.21.15.33:6805/3277300496","heartbeat_back_addr":"172.21.15.33:6809/3277300496","heartbeat_front_addr":"172.21.15.33:6807/3277300496","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6811","nonce":2439650417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6813","nonce":2439650417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6817","nonce":2439650417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6815","nonce":2439650417}]},"public_addr":"172.21.15.33:6811/2439650417","cluster_addr":"172.21.15.33:6813/2439650417","heartbeat_back_addr":"172.21.15.33:6817/2439650417","heartbeat_front_addr":"172.21.15.33:6815/2439650417","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6819","nonce":3305661464}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6821","nonce":3305661464}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6825","nonce":3305661464}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6823","nonce":3305661464}]},"public_addr":"172.21.15.33:6819/3305661464","cluster_addr":"172.21.15.33:6821/3305661464","heartbeat_back_addr":"172.21.15.33:6825/3305661464","heartbeat_front_addr":"172.21.15.33:6823/3305661464","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6827","nonce":510343171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6829","nonce":510343171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6833","nonce":510343171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6831","nonce":510343171}]},"public_addr":"172.21.15.33:6827/510343171","cluster_addr":"172.21.15.33:6829/510343171","heartbeat_back_addr":"172.21.15.33:6833/510343171","heartbeat_front_addr":"172.21.15.33:6831/510343171","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6801","nonce":1234622360}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6803","nonce":1234622360}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6807","nonce":1234622360}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6805","nonce":1234622360}]},"public_addr":"172.21.15.105:6801/1234622360","cluster_addr":"172.21.15.105:6803/1234622360","heartbeat_back_addr":"172.21.15.105:6807/1234622360","heartbeat_front_addr":"172.21.15.105:6805/1234622360","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6809","nonce":1890006054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6811","nonce":1890006054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6815","nonce":1890006054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6813","nonce":1890006054}]},"public_addr":"172.21.15.105:6809/1890006054","cluster_addr":"172.21.15.105:6811/1890006054","heartbeat_back_addr":"172.21.15.105:6815/1890006054","heartbeat_front_addr":"172.21.15.105:6813/1890006054","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6817","nonce":592734975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6819","nonce":592734975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6823","nonce":592734975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6821","nonce":592734975}]},"public_addr":"172.21.15.105:6817/592734975","cluster_addr":"172.21.15.105:6819/592734975","heartbeat_back_addr":"172.21.15.105:6823/592734975","heartbeat_front_addr":"172.21.15.105:6821/592734975","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6825","nonce":455831701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6827","nonce":455831701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6831","nonce":455831701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6829","nonce":455831701}]},"public_addr":"172.21.15.105:6825/455831701","cluster_addr":"172.21.15.105:6827/455831701","heartbeat_back_addr":"172.21.15.105:6831/455831701","heartbeat_front_addr":"172.21.15.105:6829/455831701","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2767066085' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]: dispatch 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2f", "id": [4, 3]}]: dispatch 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [4, 2]}]: dispatch 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.39", "id": [0, 1]}]: dispatch 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2767066085' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:45.627 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-09-21T09:47:45.628 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2024-09-21T09:47:45.628 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]: dispatch 2024-09-21T09:47:45.628 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2f", "id": [4, 3]}]: dispatch 2024-09-21T09:47:45.628 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [4, 2]}]: dispatch 2024-09-21T09:47:45.628 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:45 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.39", "id": [0, 1]}]: dispatch 2024-09-21T09:47:45.653 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:47:45.653 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:47:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:45 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2767066085' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:47:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:45 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-09-21T09:47:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:45 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]: dispatch 2024-09-21T09:47:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:45 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]: dispatch 2024-09-21T09:47:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:45 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2f", "id": [4, 3]}]: dispatch 2024-09-21T09:47:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:45 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [4, 2]}]: dispatch 2024-09-21T09:47:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:45 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.39", "id": [0, 1]}]: dispatch 2024-09-21T09:47:45.907 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[27969]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-09-21T09:47:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2024-09-21T09:47:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]': finished 2024-09-21T09:47:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2f", "id": [4, 3]}]': finished 2024-09-21T09:47:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [4, 2]}]': finished 2024-09-21T09:47:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.39", "id": [0, 1]}]': finished 2024-09-21T09:47:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[27969]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:47:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[39045]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-09-21T09:47:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2024-09-21T09:47:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]': finished 2024-09-21T09:47:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2f", "id": [4, 3]}]': finished 2024-09-21T09:47:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [4, 2]}]': finished 2024-09-21T09:47:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.39", "id": [0, 1]}]': finished 2024-09-21T09:47:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:46 smithi033 ceph-mon[39045]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:47:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:46 smithi105 ceph-mon[31286]: pgmap v197: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-09-21T09:47:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.b", "id": [0, 2]}]': finished 2024-09-21T09:47:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1c", "id": [5, 2]}]': finished 2024-09-21T09:47:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.2f", "id": [4, 3]}]': finished 2024-09-21T09:47:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.34", "id": [4, 2]}]': finished 2024-09-21T09:47:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:46 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.39", "id": [0, 1]}]': finished 2024-09-21T09:47:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:46 smithi105 ceph-mon[31286]: osdmap e58: 8 total, 8 up, 8 in 2024-09-21T09:47:47.670 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:47 smithi033 ceph-mon[39045]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:47:47.671 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:47 smithi033 ceph-mon[27969]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:47:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:47 smithi105 ceph-mon[31286]: osdmap e59: 8 total, 8 up, 8 in 2024-09-21T09:47:47.864 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:47.865 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-21T09:47:48.603 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2024-09-21T09:47:47.382596+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385172,"kb_used_data":6304,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576044,"statfs":{"total":767960285184,"available":765517869056,"internally_reserved":0,"allocated":6455296,"data_stored":3718874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001689"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473431+0000","last_change":"2024-09-21T09:47:24.066961+0000","last_active":"2024-09-21T09:47:45.473431+0000","last_peered":"2024-09-21T09:47:45.473431+0000","last_clean":"2024-09-21T09:47:45.473431+0000","last_became_active":"2024-09-21T09:47:24.066672+0000","last_became_peered":"2024-09-21T09:47:24.066672+0000","last_unstale":"2024-09-21T09:47:45.473431+0000","last_undegraded":"2024-09-21T09:47:45.473431+0000","last_fullsized":"2024-09-21T09:47:45.473431+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.283280+0000","last_change":"2024-09-21T09:47:24.059165+0000","last_active":"2024-09-21T09:47:25.283280+0000","last_peered":"2024-09-21T09:47:25.283280+0000","last_clean":"2024-09-21T09:47:25.283280+0000","last_became_active":"2024-09-21T09:47:24.058702+0000","last_became_peered":"2024-09-21T09:47:24.058702+0000","last_unstale":"2024-09-21T09:47:25.283280+0000","last_undegraded":"2024-09-21T09:47:25.283280+0000","last_fullsized":"2024-09-21T09:47:25.283280+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060714+0000","last_change":"2024-09-21T09:47:24.060359+0000","last_active":"2024-09-21T09:47:26.060714+0000","last_peered":"2024-09-21T09:47:26.060714+0000","last_clean":"2024-09-21T09:47:26.060714+0000","last_became_active":"2024-09-21T09:47:24.060088+0000","last_became_peered":"2024-09-21T09:47:24.060088+0000","last_unstale":"2024-09-21T09:47:26.060714+0000","last_undegraded":"2024-09-21T09:47:26.060714+0000","last_fullsized":"2024-09-21T09:47:26.060714+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473799+0000","last_change":"2024-09-21T09:47:24.065047+0000","last_active":"2024-09-21T09:47:45.473799+0000","last_peered":"2024-09-21T09:47:45.473799+0000","last_clean":"2024-09-21T09:47:45.473799+0000","last_became_active":"2024-09-21T09:47:24.064743+0000","last_became_peered":"2024-09-21T09:47:24.064743+0000","last_unstale":"2024-09-21T09:47:45.473799+0000","last_undegraded":"2024-09-21T09:47:45.473799+0000","last_fullsized":"2024-09-21T09:47:45.473799+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475055+0000","last_change":"2024-09-21T09:47:24.069692+0000","last_active":"2024-09-21T09:47:45.475055+0000","last_peered":"2024-09-21T09:47:45.475055+0000","last_clean":"2024-09-21T09:47:45.475055+0000","last_became_active":"2024-09-21T09:47:24.069090+0000","last_became_peered":"2024-09-21T09:47:24.069090+0000","last_unstale":"2024-09-21T09:47:45.475055+0000","last_undegraded":"2024-09-21T09:47:45.475055+0000","last_fullsized":"2024-09-21T09:47:45.475055+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475319+0000","last_change":"2024-09-21T09:47:24.053540+0000","last_active":"2024-09-21T09:47:45.475319+0000","last_peered":"2024-09-21T09:47:45.475319+0000","last_clean":"2024-09-21T09:47:45.475319+0000","last_became_active":"2024-09-21T09:47:24.053163+0000","last_became_peered":"2024-09-21T09:47:24.053163+0000","last_unstale":"2024-09-21T09:47:45.475319+0000","last_undegraded":"2024-09-21T09:47:45.475319+0000","last_fullsized":"2024-09-21T09:47:45.475319+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774760+0000","last_change":"2024-09-21T09:47:46.774760+0000","last_active":"2024-09-21T09:47:46.774760+0000","last_peered":"2024-09-21T09:47:46.774760+0000","last_clean":"2024-09-21T09:47:46.774760+0000","last_became_active":"2024-09-21T09:47:46.774445+0000","last_became_peered":"2024-09-21T09:47:46.774445+0000","last_unstale":"2024-09-21T09:47:46.774760+0000","last_undegraded":"2024-09-21T09:47:46.774760+0000","last_fullsized":"2024-09-21T09:47:46.774760+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062115+0000","last_change":"2024-09-21T09:47:24.064091+0000","last_active":"2024-09-21T09:47:26.062115+0000","last_peered":"2024-09-21T09:47:26.062115+0000","last_clean":"2024-09-21T09:47:26.062115+0000","last_became_active":"2024-09-21T09:47:24.063698+0000","last_became_peered":"2024-09-21T09:47:24.063698+0000","last_unstale":"2024-09-21T09:47:26.062115+0000","last_undegraded":"2024-09-21T09:47:26.062115+0000","last_fullsized":"2024-09-21T09:47:26.062115+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476570+0000","last_change":"2024-09-21T09:47:24.071747+0000","last_active":"2024-09-21T09:47:45.476570+0000","last_peered":"2024-09-21T09:47:45.476570+0000","last_clean":"2024-09-21T09:47:45.476570+0000","last_became_active":"2024-09-21T09:47:24.071475+0000","last_became_peered":"2024-09-21T09:47:24.071475+0000","last_unstale":"2024-09-21T09:47:45.476570+0000","last_undegraded":"2024-09-21T09:47:45.476570+0000","last_fullsized":"2024-09-21T09:47:45.476570+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061249+0000","last_change":"2024-09-21T09:47:24.058216+0000","last_active":"2024-09-21T09:47:26.061249+0000","last_peered":"2024-09-21T09:47:26.061249+0000","last_clean":"2024-09-21T09:47:26.061249+0000","last_became_active":"2024-09-21T09:47:24.057940+0000","last_became_peered":"2024-09-21T09:47:24.057940+0000","last_unstale":"2024-09-21T09:47:26.061249+0000","last_undegraded":"2024-09-21T09:47:26.061249+0000","last_fullsized":"2024-09-21T09:47:26.061249+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061233+0000","last_change":"2024-09-21T09:47:24.053543+0000","last_active":"2024-09-21T09:47:26.061233+0000","last_peered":"2024-09-21T09:47:26.061233+0000","last_clean":"2024-09-21T09:47:26.061233+0000","last_became_active":"2024-09-21T09:47:24.053264+0000","last_became_peered":"2024-09-21T09:47:24.053264+0000","last_unstale":"2024-09-21T09:47:26.061233+0000","last_undegraded":"2024-09-21T09:47:26.061233+0000","last_fullsized":"2024-09-21T09:47:26.061233+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":4,"reported_epoch":59,"state":"activating","last_fresh":"2024-09-21T09:47:46.473995+0000","last_change":"2024-09-21T09:47:46.473995+0000","last_active":"2024-09-21T09:47:23.027120+0000","last_peered":"2024-09-21T09:47:23.027120+0000","last_clean":"2024-09-21T09:47:23.027120+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T09:47:46.473995+0000","last_undegraded":"2024-09-21T09:47:46.473995+0000","last_fullsized":"2024-09-21T09:47:46.473995+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":["2","0","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061208+0000","last_change":"2024-09-21T09:47:24.064334+0000","last_active":"2024-09-21T09:47:26.061208+0000","last_peered":"2024-09-21T09:47:26.061208+0000","last_clean":"2024-09-21T09:47:26.061208+0000","last_became_active":"2024-09-21T09:47:24.064063+0000","last_became_peered":"2024-09-21T09:47:24.064063+0000","last_unstale":"2024-09-21T09:47:26.061208+0000","last_undegraded":"2024-09-21T09:47:26.061208+0000","last_fullsized":"2024-09-21T09:47:26.061208+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060237+0000","last_change":"2024-09-21T09:47:24.058842+0000","last_active":"2024-09-21T09:47:26.060237+0000","last_peered":"2024-09-21T09:47:26.060237+0000","last_clean":"2024-09-21T09:47:26.060237+0000","last_became_active":"2024-09-21T09:47:24.058245+0000","last_became_peered":"2024-09-21T09:47:24.058245+0000","last_unstale":"2024-09-21T09:47:26.060237+0000","last_undegraded":"2024-09-21T09:47:26.060237+0000","last_fullsized":"2024-09-21T09:47:26.060237+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282593+0000","last_change":"2024-09-21T09:47:24.052028+0000","last_active":"2024-09-21T09:47:25.282593+0000","last_peered":"2024-09-21T09:47:25.282593+0000","last_clean":"2024-09-21T09:47:25.282593+0000","last_became_active":"2024-09-21T09:47:24.051766+0000","last_became_peered":"2024-09-21T09:47:24.051766+0000","last_unstale":"2024-09-21T09:47:25.282593+0000","last_undegraded":"2024-09-21T09:47:25.282593+0000","last_fullsized":"2024-09-21T09:47:25.282593+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473412+0000","last_change":"2024-09-21T09:47:24.060117+0000","last_active":"2024-09-21T09:47:45.473412+0000","last_peered":"2024-09-21T09:47:45.473412+0000","last_clean":"2024-09-21T09:47:45.473412+0000","last_became_active":"2024-09-21T09:47:24.059852+0000","last_became_peered":"2024-09-21T09:47:24.059852+0000","last_unstale":"2024-09-21T09:47:45.473412+0000","last_undegraded":"2024-09-21T09:47:45.473412+0000","last_fullsized":"2024-09-21T09:47:45.473412+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.480400+0000","last_change":"2024-09-21T09:47:46.480400+0000","last_active":"2024-09-21T09:47:46.480400+0000","last_peered":"2024-09-21T09:47:46.480400+0000","last_clean":"2024-09-21T09:47:46.480400+0000","last_became_active":"2024-09-21T09:47:46.480103+0000","last_became_peered":"2024-09-21T09:47:46.480103+0000","last_unstale":"2024-09-21T09:47:46.480400+0000","last_undegraded":"2024-09-21T09:47:46.480400+0000","last_fullsized":"2024-09-21T09:47:46.480400+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469780+0000","last_change":"2024-09-21T09:47:24.057147+0000","last_active":"2024-09-21T09:47:45.469780+0000","last_peered":"2024-09-21T09:47:45.469780+0000","last_clean":"2024-09-21T09:47:45.469780+0000","last_became_active":"2024-09-21T09:47:24.056798+0000","last_became_peered":"2024-09-21T09:47:24.056798+0000","last_unstale":"2024-09-21T09:47:45.469780+0000","last_undegraded":"2024-09-21T09:47:45.469780+0000","last_fullsized":"2024-09-21T09:47:45.469780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475925+0000","last_change":"2024-09-21T09:47:24.070640+0000","last_active":"2024-09-21T09:47:45.475925+0000","last_peered":"2024-09-21T09:47:45.475925+0000","last_clean":"2024-09-21T09:47:45.475925+0000","last_became_active":"2024-09-21T09:47:24.070359+0000","last_became_peered":"2024-09-21T09:47:24.070359+0000","last_unstale":"2024-09-21T09:47:45.475925+0000","last_undegraded":"2024-09-21T09:47:45.475925+0000","last_fullsized":"2024-09-21T09:47:45.475925+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.058738+0000","last_change":"2024-09-21T09:47:24.056659+0000","last_active":"2024-09-21T09:47:26.058738+0000","last_peered":"2024-09-21T09:47:26.058738+0000","last_clean":"2024-09-21T09:47:26.058738+0000","last_became_active":"2024-09-21T09:47:24.056360+0000","last_became_peered":"2024-09-21T09:47:24.056360+0000","last_unstale":"2024-09-21T09:47:26.058738+0000","last_undegraded":"2024-09-21T09:47:26.058738+0000","last_fullsized":"2024-09-21T09:47:26.058738+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060802+0000","last_change":"2024-09-21T09:47:24.065154+0000","last_active":"2024-09-21T09:47:26.060802+0000","last_peered":"2024-09-21T09:47:26.060802+0000","last_clean":"2024-09-21T09:47:26.060802+0000","last_became_active":"2024-09-21T09:47:24.064231+0000","last_became_peered":"2024-09-21T09:47:24.064231+0000","last_unstale":"2024-09-21T09:47:26.060802+0000","last_undegraded":"2024-09-21T09:47:26.060802+0000","last_fullsized":"2024-09-21T09:47:26.060802+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475326+0000","last_change":"2024-09-21T09:47:24.062716+0000","last_active":"2024-09-21T09:47:45.475326+0000","last_peered":"2024-09-21T09:47:45.475326+0000","last_clean":"2024-09-21T09:47:45.475326+0000","last_became_active":"2024-09-21T09:47:24.062437+0000","last_became_peered":"2024-09-21T09:47:24.062437+0000","last_unstale":"2024-09-21T09:47:45.475326+0000","last_undegraded":"2024-09-21T09:47:45.475326+0000","last_fullsized":"2024-09-21T09:47:45.475326+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060710+0000","last_change":"2024-09-21T09:47:24.051387+0000","last_active":"2024-09-21T09:47:26.060710+0000","last_peered":"2024-09-21T09:47:26.060710+0000","last_clean":"2024-09-21T09:47:26.060710+0000","last_became_active":"2024-09-21T09:47:24.051090+0000","last_became_peered":"2024-09-21T09:47:24.051090+0000","last_unstale":"2024-09-21T09:47:26.060710+0000","last_undegraded":"2024-09-21T09:47:26.060710+0000","last_fullsized":"2024-09-21T09:47:26.060710+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473500+0000","last_change":"2024-09-21T09:47:24.058079+0000","last_active":"2024-09-21T09:47:45.473500+0000","last_peered":"2024-09-21T09:47:45.473500+0000","last_clean":"2024-09-21T09:47:45.473500+0000","last_became_active":"2024-09-21T09:47:24.057753+0000","last_became_peered":"2024-09-21T09:47:24.057753+0000","last_unstale":"2024-09-21T09:47:45.473500+0000","last_undegraded":"2024-09-21T09:47:45.473500+0000","last_fullsized":"2024-09-21T09:47:45.473500+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062275+0000","last_change":"2024-09-21T09:47:24.057220+0000","last_active":"2024-09-21T09:47:26.062275+0000","last_peered":"2024-09-21T09:47:26.062275+0000","last_clean":"2024-09-21T09:47:26.062275+0000","last_became_active":"2024-09-21T09:47:24.056915+0000","last_became_peered":"2024-09-21T09:47:24.056915+0000","last_unstale":"2024-09-21T09:47:26.062275+0000","last_undegraded":"2024-09-21T09:47:26.062275+0000","last_fullsized":"2024-09-21T09:47:26.062275+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061489+0000","last_change":"2024-09-21T09:47:24.057310+0000","last_active":"2024-09-21T09:47:26.061489+0000","last_peered":"2024-09-21T09:47:26.061489+0000","last_clean":"2024-09-21T09:47:26.061489+0000","last_became_active":"2024-09-21T09:47:24.057047+0000","last_became_peered":"2024-09-21T09:47:24.057047+0000","last_unstale":"2024-09-21T09:47:26.061489+0000","last_undegraded":"2024-09-21T09:47:26.061489+0000","last_fullsized":"2024-09-21T09:47:26.061489+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.283112+0000","last_change":"2024-09-21T09:47:24.060976+0000","last_active":"2024-09-21T09:47:25.283112+0000","last_peered":"2024-09-21T09:47:25.283112+0000","last_clean":"2024-09-21T09:47:25.283112+0000","last_became_active":"2024-09-21T09:47:24.060725+0000","last_became_peered":"2024-09-21T09:47:24.060725+0000","last_unstale":"2024-09-21T09:47:25.283112+0000","last_undegraded":"2024-09-21T09:47:25.283112+0000","last_fullsized":"2024-09-21T09:47:25.283112+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.059810+0000","last_change":"2024-09-21T09:47:24.053563+0000","last_active":"2024-09-21T09:47:26.059810+0000","last_peered":"2024-09-21T09:47:26.059810+0000","last_clean":"2024-09-21T09:47:26.059810+0000","last_became_active":"2024-09-21T09:47:24.052853+0000","last_became_peered":"2024-09-21T09:47:24.052853+0000","last_unstale":"2024-09-21T09:47:26.059810+0000","last_undegraded":"2024-09-21T09:47:26.059810+0000","last_fullsized":"2024-09-21T09:47:26.059810+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060904+0000","last_change":"2024-09-21T09:47:24.068204+0000","last_active":"2024-09-21T09:47:26.060904+0000","last_peered":"2024-09-21T09:47:26.060904+0000","last_clean":"2024-09-21T09:47:26.060904+0000","last_became_active":"2024-09-21T09:47:24.067911+0000","last_became_peered":"2024-09-21T09:47:24.067911+0000","last_unstale":"2024-09-21T09:47:26.060904+0000","last_undegraded":"2024-09-21T09:47:26.060904+0000","last_fullsized":"2024-09-21T09:47:26.060904+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475415+0000","last_change":"2024-09-21T09:47:24.058882+0000","last_active":"2024-09-21T09:47:45.475415+0000","last_peered":"2024-09-21T09:47:45.475415+0000","last_clean":"2024-09-21T09:47:45.475415+0000","last_became_active":"2024-09-21T09:47:24.058587+0000","last_became_peered":"2024-09-21T09:47:24.058587+0000","last_unstale":"2024-09-21T09:47:45.475415+0000","last_undegraded":"2024-09-21T09:47:45.475415+0000","last_fullsized":"2024-09-21T09:47:45.475415+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469392+0000","last_change":"2024-09-21T09:47:24.060446+0000","last_active":"2024-09-21T09:47:45.469392+0000","last_peered":"2024-09-21T09:47:45.469392+0000","last_clean":"2024-09-21T09:47:45.469392+0000","last_became_active":"2024-09-21T09:47:24.060165+0000","last_became_peered":"2024-09-21T09:47:24.060165+0000","last_unstale":"2024-09-21T09:47:45.469392+0000","last_undegraded":"2024-09-21T09:47:45.469392+0000","last_fullsized":"2024-09-21T09:47:45.469392+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.057761+0000","last_change":"2024-09-21T09:47:24.059974+0000","last_active":"2024-09-21T09:47:26.057761+0000","last_peered":"2024-09-21T09:47:26.057761+0000","last_clean":"2024-09-21T09:47:26.057761+0000","last_became_active":"2024-09-21T09:47:24.059270+0000","last_became_peered":"2024-09-21T09:47:24.059270+0000","last_unstale":"2024-09-21T09:47:26.057761+0000","last_undegraded":"2024-09-21T09:47:26.057761+0000","last_fullsized":"2024-09-21T09:47:26.057761+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470251+0000","last_change":"2024-09-21T09:47:24.048941+0000","last_active":"2024-09-21T09:47:45.470251+0000","last_peered":"2024-09-21T09:47:45.470251+0000","last_clean":"2024-09-21T09:47:45.470251+0000","last_became_active":"2024-09-21T09:47:24.048631+0000","last_became_peered":"2024-09-21T09:47:24.048631+0000","last_unstale":"2024-09-21T09:47:45.470251+0000","last_undegraded":"2024-09-21T09:47:45.470251+0000","last_fullsized":"2024-09-21T09:47:45.470251+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":16,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.064507+0000","last_change":"2024-09-21T09:47:20.866010+0000","last_active":"2024-09-21T09:47:26.064507+0000","last_peered":"2024-09-21T09:47:26.064507+0000","last_clean":"2024-09-21T09:47:26.064507+0000","last_became_active":"2024-09-21T09:47:20.865732+0000","last_became_peered":"2024-09-21T09:47:20.865732+0000","last_unstale":"2024-09-21T09:47:26.064507+0000","last_undegraded":"2024-09-21T09:47:26.064507+0000","last_fullsized":"2024-09-21T09:47:26.064507+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060967+0000","last_change":"2024-09-21T09:47:24.067813+0000","last_active":"2024-09-21T09:47:26.060967+0000","last_peered":"2024-09-21T09:47:26.060967+0000","last_clean":"2024-09-21T09:47:26.060967+0000","last_became_active":"2024-09-21T09:47:24.067412+0000","last_became_peered":"2024-09-21T09:47:24.067412+0000","last_unstale":"2024-09-21T09:47:26.060967+0000","last_undegraded":"2024-09-21T09:47:26.060967+0000","last_fullsized":"2024-09-21T09:47:26.060967+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":18,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-21T09:47:45.473703+0000","last_change":"2024-09-21T09:47:45.473958+0000","last_active":"2024-09-21T09:47:45.473671+0000","last_peered":"2024-09-21T09:47:26.064536+0000","last_clean":"2024-09-21T09:47:26.064536+0000","last_became_active":"2024-09-21T09:47:20.861410+0000","last_became_peered":"2024-09-21T09:47:20.861410+0000","last_unstale":"2024-09-21T09:47:45.473703+0000","last_undegraded":"2024-09-21T09:47:45.473703+0000","last_fullsized":"2024-09-21T09:47:45.473703+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:57.766502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061989+0000","last_change":"2024-09-21T09:47:20.859829+0000","last_active":"2024-09-21T09:47:26.061989+0000","last_peered":"2024-09-21T09:47:26.061989+0000","last_clean":"2024-09-21T09:47:26.061989+0000","last_became_active":"2024-09-21T09:47:20.859507+0000","last_became_peered":"2024-09-21T09:47:20.859507+0000","last_unstale":"2024-09-21T09:47:26.061989+0000","last_undegraded":"2024-09-21T09:47:26.061989+0000","last_fullsized":"2024-09-21T09:47:26.061989+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062234+0000","last_change":"2024-09-21T09:47:24.053702+0000","last_active":"2024-09-21T09:47:26.062234+0000","last_peered":"2024-09-21T09:47:26.062234+0000","last_clean":"2024-09-21T09:47:26.062234+0000","last_became_active":"2024-09-21T09:47:24.053295+0000","last_became_peered":"2024-09-21T09:47:24.053295+0000","last_unstale":"2024-09-21T09:47:26.062234+0000","last_undegraded":"2024-09-21T09:47:26.062234+0000","last_fullsized":"2024-09-21T09:47:26.062234+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475673+0000","last_change":"2024-09-21T09:47:20.863181+0000","last_active":"2024-09-21T09:47:45.475673+0000","last_peered":"2024-09-21T09:47:45.475673+0000","last_clean":"2024-09-21T09:47:45.475673+0000","last_became_active":"2024-09-21T09:47:20.862895+0000","last_became_peered":"2024-09-21T09:47:20.862895+0000","last_unstale":"2024-09-21T09:47:45.475673+0000","last_undegraded":"2024-09-21T09:47:45.475673+0000","last_fullsized":"2024-09-21T09:47:45.475673+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282439+0000","last_change":"2024-09-21T09:47:24.051248+0000","last_active":"2024-09-21T09:47:25.282439+0000","last_peered":"2024-09-21T09:47:25.282439+0000","last_clean":"2024-09-21T09:47:25.282439+0000","last_became_active":"2024-09-21T09:47:24.050918+0000","last_became_peered":"2024-09-21T09:47:24.050918+0000","last_unstale":"2024-09-21T09:47:25.282439+0000","last_undegraded":"2024-09-21T09:47:25.282439+0000","last_fullsized":"2024-09-21T09:47:25.282439+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061495+0000","last_change":"2024-09-21T09:47:20.865532+0000","last_active":"2024-09-21T09:47:26.061495+0000","last_peered":"2024-09-21T09:47:26.061495+0000","last_clean":"2024-09-21T09:47:26.061495+0000","last_became_active":"2024-09-21T09:47:20.865103+0000","last_became_peered":"2024-09-21T09:47:20.865103+0000","last_unstale":"2024-09-21T09:47:26.061495+0000","last_undegraded":"2024-09-21T09:47:26.061495+0000","last_fullsized":"2024-09-21T09:47:26.061495+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474292+0000","last_change":"2024-09-21T09:47:24.067330+0000","last_active":"2024-09-21T09:47:45.474292+0000","last_peered":"2024-09-21T09:47:45.474292+0000","last_clean":"2024-09-21T09:47:45.474292+0000","last_became_active":"2024-09-21T09:47:24.066939+0000","last_became_peered":"2024-09-21T09:47:24.066939+0000","last_unstale":"2024-09-21T09:47:45.474292+0000","last_undegraded":"2024-09-21T09:47:45.474292+0000","last_fullsized":"2024-09-21T09:47:45.474292+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474434+0000","last_change":"2024-09-21T09:47:20.860605+0000","last_active":"2024-09-21T09:47:45.474434+0000","last_peered":"2024-09-21T09:47:45.474434+0000","last_clean":"2024-09-21T09:47:45.474434+0000","last_became_active":"2024-09-21T09:47:20.860288+0000","last_became_peered":"2024-09-21T09:47:20.860288+0000","last_unstale":"2024-09-21T09:47:45.474434+0000","last_undegraded":"2024-09-21T09:47:45.474434+0000","last_fullsized":"2024-09-21T09:47:45.474434+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061722+0000","last_change":"2024-09-21T09:47:24.057860+0000","last_active":"2024-09-21T09:47:26.061722+0000","last_peered":"2024-09-21T09:47:26.061722+0000","last_clean":"2024-09-21T09:47:26.061722+0000","last_became_active":"2024-09-21T09:47:24.057594+0000","last_became_peered":"2024-09-21T09:47:24.057594+0000","last_unstale":"2024-09-21T09:47:26.061722+0000","last_undegraded":"2024-09-21T09:47:26.061722+0000","last_fullsized":"2024-09-21T09:47:26.061722+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.063141+0000","last_change":"2024-09-21T09:47:20.857026+0000","last_active":"2024-09-21T09:47:26.063141+0000","last_peered":"2024-09-21T09:47:26.063141+0000","last_clean":"2024-09-21T09:47:26.063141+0000","last_became_active":"2024-09-21T09:47:20.856719+0000","last_became_peered":"2024-09-21T09:47:20.856719+0000","last_unstale":"2024-09-21T09:47:26.063141+0000","last_undegraded":"2024-09-21T09:47:26.063141+0000","last_fullsized":"2024-09-21T09:47:26.063141+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471636+0000","last_change":"2024-09-21T09:47:24.059537+0000","last_active":"2024-09-21T09:47:45.471636+0000","last_peered":"2024-09-21T09:47:45.471636+0000","last_clean":"2024-09-21T09:47:45.471636+0000","last_became_active":"2024-09-21T09:47:24.059264+0000","last_became_peered":"2024-09-21T09:47:24.059264+0000","last_unstale":"2024-09-21T09:47:45.471636+0000","last_undegraded":"2024-09-21T09:47:45.471636+0000","last_fullsized":"2024-09-21T09:47:45.471636+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":21,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.066221+0000","last_change":"2024-09-21T09:47:20.864446+0000","last_active":"2024-09-21T09:47:26.066221+0000","last_peered":"2024-09-21T09:47:26.066221+0000","last_clean":"2024-09-21T09:47:26.066221+0000","last_became_active":"2024-09-21T09:47:20.864160+0000","last_became_peered":"2024-09-21T09:47:20.864160+0000","last_unstale":"2024-09-21T09:47:26.066221+0000","last_undegraded":"2024-09-21T09:47:26.066221+0000","last_fullsized":"2024-09-21T09:47:26.066221+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475529+0000","last_change":"2024-09-21T09:47:24.070010+0000","last_active":"2024-09-21T09:47:45.475529+0000","last_peered":"2024-09-21T09:47:45.475529+0000","last_clean":"2024-09-21T09:47:45.475529+0000","last_became_active":"2024-09-21T09:47:24.069698+0000","last_became_peered":"2024-09-21T09:47:24.069698+0000","last_unstale":"2024-09-21T09:47:45.475529+0000","last_undegraded":"2024-09-21T09:47:45.475529+0000","last_fullsized":"2024-09-21T09:47:45.475529+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.064742+0000","last_change":"2024-09-21T09:47:20.858242+0000","last_active":"2024-09-21T09:47:26.064742+0000","last_peered":"2024-09-21T09:47:26.064742+0000","last_clean":"2024-09-21T09:47:26.064742+0000","last_became_active":"2024-09-21T09:47:20.857951+0000","last_became_peered":"2024-09-21T09:47:20.857951+0000","last_unstale":"2024-09-21T09:47:26.064742+0000","last_undegraded":"2024-09-21T09:47:26.064742+0000","last_fullsized":"2024-09-21T09:47:26.064742+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060336+0000","last_change":"2024-09-21T09:47:24.063666+0000","last_active":"2024-09-21T09:47:26.060336+0000","last_peered":"2024-09-21T09:47:26.060336+0000","last_clean":"2024-09-21T09:47:26.060336+0000","last_became_active":"2024-09-21T09:47:24.063116+0000","last_became_peered":"2024-09-21T09:47:24.063116+0000","last_unstale":"2024-09-21T09:47:26.060336+0000","last_undegraded":"2024-09-21T09:47:26.060336+0000","last_fullsized":"2024-09-21T09:47:26.060336+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475534+0000","last_change":"2024-09-21T09:47:20.867134+0000","last_active":"2024-09-21T09:47:45.475534+0000","last_peered":"2024-09-21T09:47:45.475534+0000","last_clean":"2024-09-21T09:47:45.475534+0000","last_became_active":"2024-09-21T09:47:20.866875+0000","last_became_peered":"2024-09-21T09:47:20.866875+0000","last_unstale":"2024-09-21T09:47:45.475534+0000","last_undegraded":"2024-09-21T09:47:45.475534+0000","last_fullsized":"2024-09-21T09:47:45.475534+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475144+0000","last_change":"2024-09-21T09:47:24.069831+0000","last_active":"2024-09-21T09:47:45.475144+0000","last_peered":"2024-09-21T09:47:45.475144+0000","last_clean":"2024-09-21T09:47:45.475144+0000","last_became_active":"2024-09-21T09:47:24.069569+0000","last_became_peered":"2024-09-21T09:47:24.069569+0000","last_unstale":"2024-09-21T09:47:45.475144+0000","last_undegraded":"2024-09-21T09:47:45.475144+0000","last_fullsized":"2024-09-21T09:47:45.475144+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.058185+0000","last_change":"2024-09-21T09:47:20.862992+0000","last_active":"2024-09-21T09:47:26.058185+0000","last_peered":"2024-09-21T09:47:26.058185+0000","last_clean":"2024-09-21T09:47:26.058185+0000","last_became_active":"2024-09-21T09:47:20.862582+0000","last_became_peered":"2024-09-21T09:47:20.862582+0000","last_unstale":"2024-09-21T09:47:26.058185+0000","last_undegraded":"2024-09-21T09:47:26.058185+0000","last_fullsized":"2024-09-21T09:47:26.058185+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.057856+0000","last_change":"2024-09-21T09:47:24.058311+0000","last_active":"2024-09-21T09:47:26.057856+0000","last_peered":"2024-09-21T09:47:26.057856+0000","last_clean":"2024-09-21T09:47:26.057856+0000","last_became_active":"2024-09-21T09:47:24.057961+0000","last_became_peered":"2024-09-21T09:47:24.057961+0000","last_unstale":"2024-09-21T09:47:26.057856+0000","last_undegraded":"2024-09-21T09:47:26.057856+0000","last_fullsized":"2024-09-21T09:47:26.057856+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469194+0000","last_change":"2024-09-21T09:47:20.856509+0000","last_active":"2024-09-21T09:47:45.469194+0000","last_peered":"2024-09-21T09:47:45.469194+0000","last_clean":"2024-09-21T09:47:45.469194+0000","last_became_active":"2024-09-21T09:47:20.856229+0000","last_became_peered":"2024-09-21T09:47:20.856229+0000","last_unstale":"2024-09-21T09:47:45.469194+0000","last_undegraded":"2024-09-21T09:47:45.469194+0000","last_fullsized":"2024-09-21T09:47:45.469194+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474200+0000","last_change":"2024-09-21T09:47:24.069301+0000","last_active":"2024-09-21T09:47:45.474200+0000","last_peered":"2024-09-21T09:47:45.474200+0000","last_clean":"2024-09-21T09:47:45.474200+0000","last_became_active":"2024-09-21T09:47:24.068917+0000","last_became_peered":"2024-09-21T09:47:24.068917+0000","last_unstale":"2024-09-21T09:47:45.474200+0000","last_undegraded":"2024-09-21T09:47:45.474200+0000","last_fullsized":"2024-09-21T09:47:45.474200+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471246+0000","last_change":"2024-09-21T09:47:20.862118+0000","last_active":"2024-09-21T09:47:45.471246+0000","last_peered":"2024-09-21T09:47:45.471246+0000","last_clean":"2024-09-21T09:47:45.471246+0000","last_became_active":"2024-09-21T09:47:20.861815+0000","last_became_peered":"2024-09-21T09:47:20.861815+0000","last_unstale":"2024-09-21T09:47:45.471246+0000","last_undegraded":"2024-09-21T09:47:45.471246+0000","last_fullsized":"2024-09-21T09:47:45.471246+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474780+0000","last_change":"2024-09-21T09:47:24.069502+0000","last_active":"2024-09-21T09:47:45.474780+0000","last_peered":"2024-09-21T09:47:45.474780+0000","last_clean":"2024-09-21T09:47:45.474780+0000","last_became_active":"2024-09-21T09:47:24.069235+0000","last_became_peered":"2024-09-21T09:47:24.069235+0000","last_unstale":"2024-09-21T09:47:45.474780+0000","last_undegraded":"2024-09-21T09:47:45.474780+0000","last_fullsized":"2024-09-21T09:47:45.474780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469921+0000","last_change":"2024-09-21T09:47:20.862503+0000","last_active":"2024-09-21T09:47:45.469921+0000","last_peered":"2024-09-21T09:47:45.469921+0000","last_clean":"2024-09-21T09:47:45.469921+0000","last_became_active":"2024-09-21T09:47:20.862237+0000","last_became_peered":"2024-09-21T09:47:20.862237+0000","last_unstale":"2024-09-21T09:47:45.469921+0000","last_undegraded":"2024-09-21T09:47:45.469921+0000","last_fullsized":"2024-09-21T09:47:45.469921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476257+0000","last_change":"2024-09-21T09:47:24.069273+0000","last_active":"2024-09-21T09:47:45.476257+0000","last_peered":"2024-09-21T09:47:45.476257+0000","last_clean":"2024-09-21T09:47:45.476257+0000","last_became_active":"2024-09-21T09:47:24.068990+0000","last_became_peered":"2024-09-21T09:47:24.068990+0000","last_unstale":"2024-09-21T09:47:45.476257+0000","last_undegraded":"2024-09-21T09:47:45.476257+0000","last_fullsized":"2024-09-21T09:47:45.476257+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.043966+0000","last_change":"2024-09-21T09:47:20.858095+0000","last_active":"2024-09-21T09:47:26.043966+0000","last_peered":"2024-09-21T09:47:26.043966+0000","last_clean":"2024-09-21T09:47:26.043966+0000","last_became_active":"2024-09-21T09:47:20.857760+0000","last_became_peered":"2024-09-21T09:47:20.857760+0000","last_unstale":"2024-09-21T09:47:26.043966+0000","last_undegraded":"2024-09-21T09:47:26.043966+0000","last_fullsized":"2024-09-21T09:47:26.043966+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473915+0000","last_change":"2024-09-21T09:47:24.063342+0000","last_active":"2024-09-21T09:47:45.473915+0000","last_peered":"2024-09-21T09:47:45.473915+0000","last_clean":"2024-09-21T09:47:45.473915+0000","last_became_active":"2024-09-21T09:47:24.063070+0000","last_became_peered":"2024-09-21T09:47:24.063070+0000","last_unstale":"2024-09-21T09:47:45.473915+0000","last_undegraded":"2024-09-21T09:47:45.473915+0000","last_fullsized":"2024-09-21T09:47:45.473915+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469526+0000","last_change":"2024-09-21T09:47:20.859702+0000","last_active":"2024-09-21T09:47:45.469526+0000","last_peered":"2024-09-21T09:47:45.469526+0000","last_clean":"2024-09-21T09:47:45.469526+0000","last_became_active":"2024-09-21T09:47:20.859443+0000","last_became_peered":"2024-09-21T09:47:20.859443+0000","last_unstale":"2024-09-21T09:47:45.469526+0000","last_undegraded":"2024-09-21T09:47:45.469526+0000","last_fullsized":"2024-09-21T09:47:45.469526+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475617+0000","last_change":"2024-09-21T09:47:24.070338+0000","last_active":"2024-09-21T09:47:45.475617+0000","last_peered":"2024-09-21T09:47:45.475617+0000","last_clean":"2024-09-21T09:47:45.475617+0000","last_became_active":"2024-09-21T09:47:24.070074+0000","last_became_peered":"2024-09-21T09:47:24.070074+0000","last_unstale":"2024-09-21T09:47:45.475617+0000","last_undegraded":"2024-09-21T09:47:45.475617+0000","last_fullsized":"2024-09-21T09:47:45.475617+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":25,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473341+0000","last_change":"2024-09-21T09:46:57.426595+0000","last_active":"2024-09-21T09:47:45.473341+0000","last_peered":"2024-09-21T09:47:45.473341+0000","last_clean":"2024-09-21T09:47:45.473341+0000","last_became_active":"2024-09-21T09:46:57.418997+0000","last_became_peered":"2024-09-21T09:46:57.418997+0000","last_unstale":"2024-09-21T09:47:45.473341+0000","last_undegraded":"2024-09-21T09:47:45.473341+0000","last_fullsized":"2024-09-21T09:47:45.473341+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_clean_scrub_stamp":"2024-09-21T09:44:57.819716+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475502+0000","last_change":"2024-09-21T09:47:20.866500+0000","last_active":"2024-09-21T09:47:45.475502+0000","last_peered":"2024-09-21T09:47:45.475502+0000","last_clean":"2024-09-21T09:47:45.475502+0000","last_became_active":"2024-09-21T09:47:20.866224+0000","last_became_peered":"2024-09-21T09:47:20.866224+0000","last_unstale":"2024-09-21T09:47:45.475502+0000","last_undegraded":"2024-09-21T09:47:45.475502+0000","last_fullsized":"2024-09-21T09:47:45.475502+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060176+0000","last_change":"2024-09-21T09:47:24.061053+0000","last_active":"2024-09-21T09:47:26.060176+0000","last_peered":"2024-09-21T09:47:26.060176+0000","last_clean":"2024-09-21T09:47:26.060176+0000","last_became_active":"2024-09-21T09:47:24.060632+0000","last_became_peered":"2024-09-21T09:47:24.060632+0000","last_unstale":"2024-09-21T09:47:26.060176+0000","last_undegraded":"2024-09-21T09:47:26.060176+0000","last_fullsized":"2024-09-21T09:47:26.060176+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473227+0000","last_change":"2024-09-21T09:47:20.869192+0000","last_active":"2024-09-21T09:47:45.473227+0000","last_peered":"2024-09-21T09:47:45.473227+0000","last_clean":"2024-09-21T09:47:45.473227+0000","last_became_active":"2024-09-21T09:47:20.868800+0000","last_became_peered":"2024-09-21T09:47:20.868800+0000","last_unstale":"2024-09-21T09:47:45.473227+0000","last_undegraded":"2024-09-21T09:47:45.473227+0000","last_fullsized":"2024-09-21T09:47:45.473227+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.059857+0000","last_change":"2024-09-21T09:47:24.056658+0000","last_active":"2024-09-21T09:47:26.059857+0000","last_peered":"2024-09-21T09:47:26.059857+0000","last_clean":"2024-09-21T09:47:26.059857+0000","last_became_active":"2024-09-21T09:47:24.056366+0000","last_became_peered":"2024-09-21T09:47:24.056366+0000","last_unstale":"2024-09-21T09:47:26.059857+0000","last_undegraded":"2024-09-21T09:47:26.059857+0000","last_fullsized":"2024-09-21T09:47:26.059857+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469527+0000","last_change":"2024-09-21T09:47:20.861899+0000","last_active":"2024-09-21T09:47:45.469527+0000","last_peered":"2024-09-21T09:47:45.469527+0000","last_clean":"2024-09-21T09:47:45.469527+0000","last_became_active":"2024-09-21T09:47:20.861684+0000","last_became_peered":"2024-09-21T09:47:20.861684+0000","last_unstale":"2024-09-21T09:47:45.469527+0000","last_undegraded":"2024-09-21T09:47:45.469527+0000","last_fullsized":"2024-09-21T09:47:45.469527+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282439+0000","last_change":"2024-09-21T09:47:24.058235+0000","last_active":"2024-09-21T09:47:25.282439+0000","last_peered":"2024-09-21T09:47:25.282439+0000","last_clean":"2024-09-21T09:47:25.282439+0000","last_became_active":"2024-09-21T09:47:24.057972+0000","last_became_peered":"2024-09-21T09:47:24.057972+0000","last_unstale":"2024-09-21T09:47:25.282439+0000","last_undegraded":"2024-09-21T09:47:25.282439+0000","last_fullsized":"2024-09-21T09:47:25.282439+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475569+0000","last_change":"2024-09-21T09:47:20.861547+0000","last_active":"2024-09-21T09:47:45.475569+0000","last_peered":"2024-09-21T09:47:45.475569+0000","last_clean":"2024-09-21T09:47:45.475569+0000","last_became_active":"2024-09-21T09:47:20.861254+0000","last_became_peered":"2024-09-21T09:47:20.861254+0000","last_unstale":"2024-09-21T09:47:45.475569+0000","last_undegraded":"2024-09-21T09:47:45.475569+0000","last_fullsized":"2024-09-21T09:47:45.475569+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471048+0000","last_change":"2024-09-21T09:47:24.060191+0000","last_active":"2024-09-21T09:47:45.471048+0000","last_peered":"2024-09-21T09:47:45.471048+0000","last_clean":"2024-09-21T09:47:45.471048+0000","last_became_active":"2024-09-21T09:47:24.059929+0000","last_became_peered":"2024-09-21T09:47:24.059929+0000","last_unstale":"2024-09-21T09:47:45.471048+0000","last_undegraded":"2024-09-21T09:47:45.471048+0000","last_fullsized":"2024-09-21T09:47:45.471048+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282787+0000","last_change":"2024-09-21T09:47:20.862181+0000","last_active":"2024-09-21T09:47:25.282787+0000","last_peered":"2024-09-21T09:47:25.282787+0000","last_clean":"2024-09-21T09:47:25.282787+0000","last_became_active":"2024-09-21T09:47:20.861903+0000","last_became_peered":"2024-09-21T09:47:20.861903+0000","last_unstale":"2024-09-21T09:47:25.282787+0000","last_undegraded":"2024-09-21T09:47:25.282787+0000","last_fullsized":"2024-09-21T09:47:25.282787+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474023+0000","last_change":"2024-09-21T09:47:24.058849+0000","last_active":"2024-09-21T09:47:45.474023+0000","last_peered":"2024-09-21T09:47:45.474023+0000","last_clean":"2024-09-21T09:47:45.474023+0000","last_became_active":"2024-09-21T09:47:24.058564+0000","last_became_peered":"2024-09-21T09:47:24.058564+0000","last_unstale":"2024-09-21T09:47:45.474023+0000","last_undegraded":"2024-09-21T09:47:45.474023+0000","last_fullsized":"2024-09-21T09:47:45.474023+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476182+0000","last_change":"2024-09-21T09:47:20.866885+0000","last_active":"2024-09-21T09:47:45.476182+0000","last_peered":"2024-09-21T09:47:45.476182+0000","last_clean":"2024-09-21T09:47:45.476182+0000","last_became_active":"2024-09-21T09:47:20.866436+0000","last_became_peered":"2024-09-21T09:47:20.866436+0000","last_unstale":"2024-09-21T09:47:45.476182+0000","last_undegraded":"2024-09-21T09:47:45.476182+0000","last_fullsized":"2024-09-21T09:47:45.476182+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282714+0000","last_change":"2024-09-21T09:47:24.060367+0000","last_active":"2024-09-21T09:47:25.282714+0000","last_peered":"2024-09-21T09:47:25.282714+0000","last_clean":"2024-09-21T09:47:25.282714+0000","last_became_active":"2024-09-21T09:47:24.060083+0000","last_became_peered":"2024-09-21T09:47:24.060083+0000","last_unstale":"2024-09-21T09:47:25.282714+0000","last_undegraded":"2024-09-21T09:47:25.282714+0000","last_fullsized":"2024-09-21T09:47:25.282714+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.044158+0000","last_change":"2024-09-21T09:47:20.858947+0000","last_active":"2024-09-21T09:47:26.044158+0000","last_peered":"2024-09-21T09:47:26.044158+0000","last_clean":"2024-09-21T09:47:26.044158+0000","last_became_active":"2024-09-21T09:47:20.858680+0000","last_became_peered":"2024-09-21T09:47:20.858680+0000","last_unstale":"2024-09-21T09:47:26.044158+0000","last_undegraded":"2024-09-21T09:47:26.044158+0000","last_fullsized":"2024-09-21T09:47:26.044158+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062409+0000","last_change":"2024-09-21T09:47:24.060498+0000","last_active":"2024-09-21T09:47:26.062409+0000","last_peered":"2024-09-21T09:47:26.062409+0000","last_clean":"2024-09-21T09:47:26.062409+0000","last_became_active":"2024-09-21T09:47:24.060179+0000","last_became_peered":"2024-09-21T09:47:24.060179+0000","last_unstale":"2024-09-21T09:47:26.062409+0000","last_undegraded":"2024-09-21T09:47:26.062409+0000","last_fullsized":"2024-09-21T09:47:26.062409+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.064735+0000","last_change":"2024-09-21T09:47:20.865797+0000","last_active":"2024-09-21T09:47:26.064735+0000","last_peered":"2024-09-21T09:47:26.064735+0000","last_clean":"2024-09-21T09:47:26.064735+0000","last_became_active":"2024-09-21T09:47:20.865466+0000","last_became_peered":"2024-09-21T09:47:20.865466+0000","last_unstale":"2024-09-21T09:47:26.064735+0000","last_undegraded":"2024-09-21T09:47:26.064735+0000","last_fullsized":"2024-09-21T09:47:26.064735+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060421+0000","last_change":"2024-09-21T09:47:24.058690+0000","last_active":"2024-09-21T09:47:26.060421+0000","last_peered":"2024-09-21T09:47:26.060421+0000","last_clean":"2024-09-21T09:47:26.060421+0000","last_became_active":"2024-09-21T09:47:24.058371+0000","last_became_peered":"2024-09-21T09:47:24.058371+0000","last_unstale":"2024-09-21T09:47:26.060421+0000","last_undegraded":"2024-09-21T09:47:26.060421+0000","last_fullsized":"2024-09-21T09:47:26.060421+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473115+0000","last_change":"2024-09-21T09:47:20.868994+0000","last_active":"2024-09-21T09:47:45.473115+0000","last_peered":"2024-09-21T09:47:45.473115+0000","last_clean":"2024-09-21T09:47:45.473115+0000","last_became_active":"2024-09-21T09:47:20.868686+0000","last_became_peered":"2024-09-21T09:47:20.868686+0000","last_unstale":"2024-09-21T09:47:45.473115+0000","last_undegraded":"2024-09-21T09:47:45.473115+0000","last_fullsized":"2024-09-21T09:47:45.473115+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470397+0000","last_change":"2024-09-21T09:47:24.055068+0000","last_active":"2024-09-21T09:47:45.470397+0000","last_peered":"2024-09-21T09:47:45.470397+0000","last_clean":"2024-09-21T09:47:45.470397+0000","last_became_active":"2024-09-21T09:47:24.054785+0000","last_became_peered":"2024-09-21T09:47:24.054785+0000","last_unstale":"2024-09-21T09:47:45.470397+0000","last_undegraded":"2024-09-21T09:47:45.470397+0000","last_fullsized":"2024-09-21T09:47:45.470397+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282677+0000","last_change":"2024-09-21T09:47:20.862674+0000","last_active":"2024-09-21T09:47:25.282677+0000","last_peered":"2024-09-21T09:47:25.282677+0000","last_clean":"2024-09-21T09:47:25.282677+0000","last_became_active":"2024-09-21T09:47:20.861988+0000","last_became_peered":"2024-09-21T09:47:20.861988+0000","last_unstale":"2024-09-21T09:47:25.282677+0000","last_undegraded":"2024-09-21T09:47:25.282677+0000","last_fullsized":"2024-09-21T09:47:25.282677+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062272+0000","last_change":"2024-09-21T09:47:24.058826+0000","last_active":"2024-09-21T09:47:26.062272+0000","last_peered":"2024-09-21T09:47:26.062272+0000","last_clean":"2024-09-21T09:47:26.062272+0000","last_became_active":"2024-09-21T09:47:24.058565+0000","last_became_peered":"2024-09-21T09:47:24.058565+0000","last_unstale":"2024-09-21T09:47:26.062272+0000","last_undegraded":"2024-09-21T09:47:26.062272+0000","last_fullsized":"2024-09-21T09:47:26.062272+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.283010+0000","last_change":"2024-09-21T09:47:20.861142+0000","last_active":"2024-09-21T09:47:25.283010+0000","last_peered":"2024-09-21T09:47:25.283010+0000","last_clean":"2024-09-21T09:47:25.283010+0000","last_became_active":"2024-09-21T09:47:20.860916+0000","last_became_peered":"2024-09-21T09:47:20.860916+0000","last_unstale":"2024-09-21T09:47:25.283010+0000","last_undegraded":"2024-09-21T09:47:25.283010+0000","last_fullsized":"2024-09-21T09:47:25.283010+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-21T09:47:45.470626+0000","last_change":"2024-09-21T09:47:45.470771+0000","last_active":"2024-09-21T09:47:45.470595+0000","last_peered":"2024-09-21T09:47:26.061291+0000","last_clean":"2024-09-21T09:47:26.061291+0000","last_became_active":"2024-09-21T09:47:24.053945+0000","last_became_peered":"2024-09-21T09:47:24.053945+0000","last_unstale":"2024-09-21T09:47:45.470626+0000","last_undegraded":"2024-09-21T09:47:45.470626+0000","last_fullsized":"2024-09-21T09:47:45.470626+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:50.863157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,4],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475414+0000","last_change":"2024-09-21T09:47:20.865234+0000","last_active":"2024-09-21T09:47:45.475414+0000","last_peered":"2024-09-21T09:47:45.475414+0000","last_clean":"2024-09-21T09:47:45.475414+0000","last_became_active":"2024-09-21T09:47:20.863790+0000","last_became_peered":"2024-09-21T09:47:20.863790+0000","last_unstale":"2024-09-21T09:47:45.475414+0000","last_undegraded":"2024-09-21T09:47:45.475414+0000","last_fullsized":"2024-09-21T09:47:45.475414+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060712+0000","last_change":"2024-09-21T09:47:24.056216+0000","last_active":"2024-09-21T09:47:26.060712+0000","last_peered":"2024-09-21T09:47:26.060712+0000","last_clean":"2024-09-21T09:47:26.060712+0000","last_became_active":"2024-09-21T09:47:24.055928+0000","last_became_peered":"2024-09-21T09:47:24.055928+0000","last_unstale":"2024-09-21T09:47:26.060712+0000","last_undegraded":"2024-09-21T09:47:26.060712+0000","last_fullsized":"2024-09-21T09:47:26.060712+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473935+0000","last_change":"2024-09-21T09:47:24.063533+0000","last_active":"2024-09-21T09:47:45.473935+0000","last_peered":"2024-09-21T09:47:45.473935+0000","last_clean":"2024-09-21T09:47:45.473935+0000","last_became_active":"2024-09-21T09:47:24.063293+0000","last_became_peered":"2024-09-21T09:47:24.063293+0000","last_unstale":"2024-09-21T09:47:45.473935+0000","last_undegraded":"2024-09-21T09:47:45.473935+0000","last_fullsized":"2024-09-21T09:47:45.473935+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474478+0000","last_change":"2024-09-21T09:47:20.864537+0000","last_active":"2024-09-21T09:47:45.474478+0000","last_peered":"2024-09-21T09:47:45.474478+0000","last_clean":"2024-09-21T09:47:45.474478+0000","last_became_active":"2024-09-21T09:47:20.864212+0000","last_became_peered":"2024-09-21T09:47:20.864212+0000","last_unstale":"2024-09-21T09:47:45.474478+0000","last_undegraded":"2024-09-21T09:47:45.474478+0000","last_fullsized":"2024-09-21T09:47:45.474478+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774197+0000","last_change":"2024-09-21T09:47:46.774197+0000","last_active":"2024-09-21T09:47:46.774197+0000","last_peered":"2024-09-21T09:47:46.774197+0000","last_clean":"2024-09-21T09:47:46.774197+0000","last_became_active":"2024-09-21T09:47:46.773846+0000","last_became_peered":"2024-09-21T09:47:46.773846+0000","last_unstale":"2024-09-21T09:47:46.774197+0000","last_undegraded":"2024-09-21T09:47:46.774197+0000","last_fullsized":"2024-09-21T09:47:46.774197+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475812+0000","last_change":"2024-09-21T09:47:20.866709+0000","last_active":"2024-09-21T09:47:45.475812+0000","last_peered":"2024-09-21T09:47:45.475812+0000","last_clean":"2024-09-21T09:47:45.475812+0000","last_became_active":"2024-09-21T09:47:20.866026+0000","last_became_peered":"2024-09-21T09:47:20.866026+0000","last_unstale":"2024-09-21T09:47:45.475812+0000","last_undegraded":"2024-09-21T09:47:45.475812+0000","last_fullsized":"2024-09-21T09:47:45.475812+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061633+0000","last_change":"2024-09-21T09:47:24.060801+0000","last_active":"2024-09-21T09:47:26.061633+0000","last_peered":"2024-09-21T09:47:26.061633+0000","last_clean":"2024-09-21T09:47:26.061633+0000","last_became_active":"2024-09-21T09:47:24.060536+0000","last_became_peered":"2024-09-21T09:47:24.060536+0000","last_unstale":"2024-09-21T09:47:26.061633+0000","last_undegraded":"2024-09-21T09:47:26.061633+0000","last_fullsized":"2024-09-21T09:47:26.061633+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":16,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.063670+0000","last_change":"2024-09-21T09:47:20.864529+0000","last_active":"2024-09-21T09:47:26.063670+0000","last_peered":"2024-09-21T09:47:26.063670+0000","last_clean":"2024-09-21T09:47:26.063670+0000","last_became_active":"2024-09-21T09:47:20.864185+0000","last_became_peered":"2024-09-21T09:47:20.864185+0000","last_unstale":"2024-09-21T09:47:26.063670+0000","last_undegraded":"2024-09-21T09:47:26.063670+0000","last_fullsized":"2024-09-21T09:47:26.063670+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.059837+0000","last_change":"2024-09-21T09:47:24.065677+0000","last_active":"2024-09-21T09:47:26.059837+0000","last_peered":"2024-09-21T09:47:26.059837+0000","last_clean":"2024-09-21T09:47:26.059837+0000","last_became_active":"2024-09-21T09:47:24.065373+0000","last_became_peered":"2024-09-21T09:47:24.065373+0000","last_unstale":"2024-09-21T09:47:26.059837+0000","last_undegraded":"2024-09-21T09:47:26.059837+0000","last_fullsized":"2024-09-21T09:47:26.059837+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":19,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.065338+0000","last_change":"2024-09-21T09:47:20.865947+0000","last_active":"2024-09-21T09:47:26.065338+0000","last_peered":"2024-09-21T09:47:26.065338+0000","last_clean":"2024-09-21T09:47:26.065338+0000","last_became_active":"2024-09-21T09:47:20.865683+0000","last_became_peered":"2024-09-21T09:47:20.865683+0000","last_unstale":"2024-09-21T09:47:26.065338+0000","last_undegraded":"2024-09-21T09:47:26.065338+0000","last_fullsized":"2024-09-21T09:47:26.065338+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430221,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":835428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49399999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]}]},{"osd":6,"up_from":42,"seq":180388626449,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":835829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.57499999999999996}]},{"osd":1,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.63600000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.60799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64900000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822678,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":245795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.54600000000000004}]},{"osd":1,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.56200000000000006}]},{"osd":2,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.376}]},{"osd":4,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:47:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.57499999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018907,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":606208,"data_stored":251705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.56299999999999994,"15min":0.57299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68999999999999995,"5min":0.748,"15min":0.748},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.432,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.67400000000000004}]},{"osd":1,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.38,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.65600000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.435,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67700000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.71599999999999997,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.51900000000000002,"15min":0.51600000000000001},"min":{"1min":0.42399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.624,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.60499999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.51800000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.64200000000000002,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.44900000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.68300000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:47:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.56999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]}]},{"osd":3,"up_from":25,"seq":107374182432,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":581632,"data_stored":239134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.39800000000000002,"15min":0.40000000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.55500000000000005,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.251},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41299999999999998,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.59499999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.34399999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44500000000000001,"15min":0.45000000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42799999999999999,"15min":0.43099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.69299999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65600000000000003,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:46:43 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.44600000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:47:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.498},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.45400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":244468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.35999999999999999,"15min":0.35399999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.246},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.27300000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.57799999999999996,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59999999999999998,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.442,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63800000000000001,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:47:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.46800000000000003,"15min":0.47499999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.56200000000000006,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63900000000000001,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.27100000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:47:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73099999999999998,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.75900000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.71099999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.627}]},{"osd":6,"last update":"Sat Sep 21 09:47:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.79200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":560,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":573440,"data_stored":237953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.34799999999999998,"15min":0.32400000000000001},"min":{"1min":0.24399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.52500000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.435},{"interface":"front","average":{"1min":0.379,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.28199999999999997,"5min":0.214,"15min":0.214},"max":{"1min":0.47899999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.29199999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.375,"15min":0.375},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.49399999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.41099999999999998,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.63500000000000001,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:47:22 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57299999999999995,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.65000000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71199999999999997,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.67600000000000005,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.70099999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.747}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72399999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738414,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1144,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1171456,"data_stored":828562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:47:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.33800000000000002,"15min":0.316},"min":{"1min":0.24199999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.374,"15min":0.34999999999999998},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:47:01 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.27500000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.59099999999999997,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.39500000000000002,"15min":0.38500000000000001},"min":{"1min":0.33300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Sep 21 09:47:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.317},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.497},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.33100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:47:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.56000000000000005,"15min":0.55200000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.629,"5min":0.57799999999999996,"15min":0.56999999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.65300000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:47:06 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Sep 21 09:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.621}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:47:48.608 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:47:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:48 smithi033 ceph-mon[27969]: pgmap v200: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:48 smithi033 ceph-mon[39045]: pgmap v200: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:48 smithi105 ceph-mon[31286]: pgmap v200: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:48.867 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:49.672 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:49 smithi033 ceph-mon[39045]: from='client.14493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:49.672 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:49 smithi033 ceph-mon[27969]: from='client.14493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:49.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:49 smithi105 ceph-mon[31286]: from='client.14493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:50.677 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:50.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:50 smithi033 ceph-mon[39045]: pgmap v201: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:50.678 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:50 smithi033 ceph-mon[27969]: pgmap v201: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:50.681 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-21T09:47:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:50 smithi105 ceph-mon[31286]: pgmap v201: 97 pgs: 1 activating, 2 peering, 94 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:51.419 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":201,"stamp":"2024-09-21T09:47:49.383145+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385172,"kb_used_data":6304,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576044,"statfs":{"total":767960285184,"available":765517869056,"internally_reserved":0,"allocated":6455296,"data_stored":3718874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001845"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473431+0000","last_change":"2024-09-21T09:47:24.066961+0000","last_active":"2024-09-21T09:47:45.473431+0000","last_peered":"2024-09-21T09:47:45.473431+0000","last_clean":"2024-09-21T09:47:45.473431+0000","last_became_active":"2024-09-21T09:47:24.066672+0000","last_became_peered":"2024-09-21T09:47:24.066672+0000","last_unstale":"2024-09-21T09:47:45.473431+0000","last_undegraded":"2024-09-21T09:47:45.473431+0000","last_fullsized":"2024-09-21T09:47:45.473431+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.283280+0000","last_change":"2024-09-21T09:47:24.059165+0000","last_active":"2024-09-21T09:47:25.283280+0000","last_peered":"2024-09-21T09:47:25.283280+0000","last_clean":"2024-09-21T09:47:25.283280+0000","last_became_active":"2024-09-21T09:47:24.058702+0000","last_became_peered":"2024-09-21T09:47:24.058702+0000","last_unstale":"2024-09-21T09:47:25.283280+0000","last_undegraded":"2024-09-21T09:47:25.283280+0000","last_fullsized":"2024-09-21T09:47:25.283280+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060714+0000","last_change":"2024-09-21T09:47:24.060359+0000","last_active":"2024-09-21T09:47:26.060714+0000","last_peered":"2024-09-21T09:47:26.060714+0000","last_clean":"2024-09-21T09:47:26.060714+0000","last_became_active":"2024-09-21T09:47:24.060088+0000","last_became_peered":"2024-09-21T09:47:24.060088+0000","last_unstale":"2024-09-21T09:47:26.060714+0000","last_undegraded":"2024-09-21T09:47:26.060714+0000","last_fullsized":"2024-09-21T09:47:26.060714+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473799+0000","last_change":"2024-09-21T09:47:24.065047+0000","last_active":"2024-09-21T09:47:45.473799+0000","last_peered":"2024-09-21T09:47:45.473799+0000","last_clean":"2024-09-21T09:47:45.473799+0000","last_became_active":"2024-09-21T09:47:24.064743+0000","last_became_peered":"2024-09-21T09:47:24.064743+0000","last_unstale":"2024-09-21T09:47:45.473799+0000","last_undegraded":"2024-09-21T09:47:45.473799+0000","last_fullsized":"2024-09-21T09:47:45.473799+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475055+0000","last_change":"2024-09-21T09:47:24.069692+0000","last_active":"2024-09-21T09:47:45.475055+0000","last_peered":"2024-09-21T09:47:45.475055+0000","last_clean":"2024-09-21T09:47:45.475055+0000","last_became_active":"2024-09-21T09:47:24.069090+0000","last_became_peered":"2024-09-21T09:47:24.069090+0000","last_unstale":"2024-09-21T09:47:45.475055+0000","last_undegraded":"2024-09-21T09:47:45.475055+0000","last_fullsized":"2024-09-21T09:47:45.475055+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475319+0000","last_change":"2024-09-21T09:47:24.053540+0000","last_active":"2024-09-21T09:47:45.475319+0000","last_peered":"2024-09-21T09:47:45.475319+0000","last_clean":"2024-09-21T09:47:45.475319+0000","last_became_active":"2024-09-21T09:47:24.053163+0000","last_became_peered":"2024-09-21T09:47:24.053163+0000","last_unstale":"2024-09-21T09:47:45.475319+0000","last_undegraded":"2024-09-21T09:47:45.475319+0000","last_fullsized":"2024-09-21T09:47:45.475319+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774760+0000","last_change":"2024-09-21T09:47:46.774760+0000","last_active":"2024-09-21T09:47:46.774760+0000","last_peered":"2024-09-21T09:47:46.774760+0000","last_clean":"2024-09-21T09:47:46.774760+0000","last_became_active":"2024-09-21T09:47:46.774445+0000","last_became_peered":"2024-09-21T09:47:46.774445+0000","last_unstale":"2024-09-21T09:47:46.774760+0000","last_undegraded":"2024-09-21T09:47:46.774760+0000","last_fullsized":"2024-09-21T09:47:46.774760+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062115+0000","last_change":"2024-09-21T09:47:24.064091+0000","last_active":"2024-09-21T09:47:26.062115+0000","last_peered":"2024-09-21T09:47:26.062115+0000","last_clean":"2024-09-21T09:47:26.062115+0000","last_became_active":"2024-09-21T09:47:24.063698+0000","last_became_peered":"2024-09-21T09:47:24.063698+0000","last_unstale":"2024-09-21T09:47:26.062115+0000","last_undegraded":"2024-09-21T09:47:26.062115+0000","last_fullsized":"2024-09-21T09:47:26.062115+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476570+0000","last_change":"2024-09-21T09:47:24.071747+0000","last_active":"2024-09-21T09:47:45.476570+0000","last_peered":"2024-09-21T09:47:45.476570+0000","last_clean":"2024-09-21T09:47:45.476570+0000","last_became_active":"2024-09-21T09:47:24.071475+0000","last_became_peered":"2024-09-21T09:47:24.071475+0000","last_unstale":"2024-09-21T09:47:45.476570+0000","last_undegraded":"2024-09-21T09:47:45.476570+0000","last_fullsized":"2024-09-21T09:47:45.476570+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061249+0000","last_change":"2024-09-21T09:47:24.058216+0000","last_active":"2024-09-21T09:47:26.061249+0000","last_peered":"2024-09-21T09:47:26.061249+0000","last_clean":"2024-09-21T09:47:26.061249+0000","last_became_active":"2024-09-21T09:47:24.057940+0000","last_became_peered":"2024-09-21T09:47:24.057940+0000","last_unstale":"2024-09-21T09:47:26.061249+0000","last_undegraded":"2024-09-21T09:47:26.061249+0000","last_fullsized":"2024-09-21T09:47:26.061249+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061233+0000","last_change":"2024-09-21T09:47:24.053543+0000","last_active":"2024-09-21T09:47:26.061233+0000","last_peered":"2024-09-21T09:47:26.061233+0000","last_clean":"2024-09-21T09:47:26.061233+0000","last_became_active":"2024-09-21T09:47:24.053264+0000","last_became_peered":"2024-09-21T09:47:24.053264+0000","last_unstale":"2024-09-21T09:47:26.061233+0000","last_undegraded":"2024-09-21T09:47:26.061233+0000","last_fullsized":"2024-09-21T09:47:26.061233+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":4,"reported_epoch":59,"state":"activating","last_fresh":"2024-09-21T09:47:46.473995+0000","last_change":"2024-09-21T09:47:46.473995+0000","last_active":"2024-09-21T09:47:23.027120+0000","last_peered":"2024-09-21T09:47:23.027120+0000","last_clean":"2024-09-21T09:47:23.027120+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2024-09-21T09:47:46.473995+0000","last_undegraded":"2024-09-21T09:47:46.473995+0000","last_fullsized":"2024-09-21T09:47:46.473995+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":["2","0","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061208+0000","last_change":"2024-09-21T09:47:24.064334+0000","last_active":"2024-09-21T09:47:26.061208+0000","last_peered":"2024-09-21T09:47:26.061208+0000","last_clean":"2024-09-21T09:47:26.061208+0000","last_became_active":"2024-09-21T09:47:24.064063+0000","last_became_peered":"2024-09-21T09:47:24.064063+0000","last_unstale":"2024-09-21T09:47:26.061208+0000","last_undegraded":"2024-09-21T09:47:26.061208+0000","last_fullsized":"2024-09-21T09:47:26.061208+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060237+0000","last_change":"2024-09-21T09:47:24.058842+0000","last_active":"2024-09-21T09:47:26.060237+0000","last_peered":"2024-09-21T09:47:26.060237+0000","last_clean":"2024-09-21T09:47:26.060237+0000","last_became_active":"2024-09-21T09:47:24.058245+0000","last_became_peered":"2024-09-21T09:47:24.058245+0000","last_unstale":"2024-09-21T09:47:26.060237+0000","last_undegraded":"2024-09-21T09:47:26.060237+0000","last_fullsized":"2024-09-21T09:47:26.060237+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282593+0000","last_change":"2024-09-21T09:47:24.052028+0000","last_active":"2024-09-21T09:47:25.282593+0000","last_peered":"2024-09-21T09:47:25.282593+0000","last_clean":"2024-09-21T09:47:25.282593+0000","last_became_active":"2024-09-21T09:47:24.051766+0000","last_became_peered":"2024-09-21T09:47:24.051766+0000","last_unstale":"2024-09-21T09:47:25.282593+0000","last_undegraded":"2024-09-21T09:47:25.282593+0000","last_fullsized":"2024-09-21T09:47:25.282593+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473412+0000","last_change":"2024-09-21T09:47:24.060117+0000","last_active":"2024-09-21T09:47:45.473412+0000","last_peered":"2024-09-21T09:47:45.473412+0000","last_clean":"2024-09-21T09:47:45.473412+0000","last_became_active":"2024-09-21T09:47:24.059852+0000","last_became_peered":"2024-09-21T09:47:24.059852+0000","last_unstale":"2024-09-21T09:47:45.473412+0000","last_undegraded":"2024-09-21T09:47:45.473412+0000","last_fullsized":"2024-09-21T09:47:45.473412+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.480400+0000","last_change":"2024-09-21T09:47:46.480400+0000","last_active":"2024-09-21T09:47:46.480400+0000","last_peered":"2024-09-21T09:47:46.480400+0000","last_clean":"2024-09-21T09:47:46.480400+0000","last_became_active":"2024-09-21T09:47:46.480103+0000","last_became_peered":"2024-09-21T09:47:46.480103+0000","last_unstale":"2024-09-21T09:47:46.480400+0000","last_undegraded":"2024-09-21T09:47:46.480400+0000","last_fullsized":"2024-09-21T09:47:46.480400+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469780+0000","last_change":"2024-09-21T09:47:24.057147+0000","last_active":"2024-09-21T09:47:45.469780+0000","last_peered":"2024-09-21T09:47:45.469780+0000","last_clean":"2024-09-21T09:47:45.469780+0000","last_became_active":"2024-09-21T09:47:24.056798+0000","last_became_peered":"2024-09-21T09:47:24.056798+0000","last_unstale":"2024-09-21T09:47:45.469780+0000","last_undegraded":"2024-09-21T09:47:45.469780+0000","last_fullsized":"2024-09-21T09:47:45.469780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475925+0000","last_change":"2024-09-21T09:47:24.070640+0000","last_active":"2024-09-21T09:47:45.475925+0000","last_peered":"2024-09-21T09:47:45.475925+0000","last_clean":"2024-09-21T09:47:45.475925+0000","last_became_active":"2024-09-21T09:47:24.070359+0000","last_became_peered":"2024-09-21T09:47:24.070359+0000","last_unstale":"2024-09-21T09:47:45.475925+0000","last_undegraded":"2024-09-21T09:47:45.475925+0000","last_fullsized":"2024-09-21T09:47:45.475925+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.058738+0000","last_change":"2024-09-21T09:47:24.056659+0000","last_active":"2024-09-21T09:47:26.058738+0000","last_peered":"2024-09-21T09:47:26.058738+0000","last_clean":"2024-09-21T09:47:26.058738+0000","last_became_active":"2024-09-21T09:47:24.056360+0000","last_became_peered":"2024-09-21T09:47:24.056360+0000","last_unstale":"2024-09-21T09:47:26.058738+0000","last_undegraded":"2024-09-21T09:47:26.058738+0000","last_fullsized":"2024-09-21T09:47:26.058738+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060802+0000","last_change":"2024-09-21T09:47:24.065154+0000","last_active":"2024-09-21T09:47:26.060802+0000","last_peered":"2024-09-21T09:47:26.060802+0000","last_clean":"2024-09-21T09:47:26.060802+0000","last_became_active":"2024-09-21T09:47:24.064231+0000","last_became_peered":"2024-09-21T09:47:24.064231+0000","last_unstale":"2024-09-21T09:47:26.060802+0000","last_undegraded":"2024-09-21T09:47:26.060802+0000","last_fullsized":"2024-09-21T09:47:26.060802+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475326+0000","last_change":"2024-09-21T09:47:24.062716+0000","last_active":"2024-09-21T09:47:45.475326+0000","last_peered":"2024-09-21T09:47:45.475326+0000","last_clean":"2024-09-21T09:47:45.475326+0000","last_became_active":"2024-09-21T09:47:24.062437+0000","last_became_peered":"2024-09-21T09:47:24.062437+0000","last_unstale":"2024-09-21T09:47:45.475326+0000","last_undegraded":"2024-09-21T09:47:45.475326+0000","last_fullsized":"2024-09-21T09:47:45.475326+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060710+0000","last_change":"2024-09-21T09:47:24.051387+0000","last_active":"2024-09-21T09:47:26.060710+0000","last_peered":"2024-09-21T09:47:26.060710+0000","last_clean":"2024-09-21T09:47:26.060710+0000","last_became_active":"2024-09-21T09:47:24.051090+0000","last_became_peered":"2024-09-21T09:47:24.051090+0000","last_unstale":"2024-09-21T09:47:26.060710+0000","last_undegraded":"2024-09-21T09:47:26.060710+0000","last_fullsized":"2024-09-21T09:47:26.060710+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473500+0000","last_change":"2024-09-21T09:47:24.058079+0000","last_active":"2024-09-21T09:47:45.473500+0000","last_peered":"2024-09-21T09:47:45.473500+0000","last_clean":"2024-09-21T09:47:45.473500+0000","last_became_active":"2024-09-21T09:47:24.057753+0000","last_became_peered":"2024-09-21T09:47:24.057753+0000","last_unstale":"2024-09-21T09:47:45.473500+0000","last_undegraded":"2024-09-21T09:47:45.473500+0000","last_fullsized":"2024-09-21T09:47:45.473500+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062275+0000","last_change":"2024-09-21T09:47:24.057220+0000","last_active":"2024-09-21T09:47:26.062275+0000","last_peered":"2024-09-21T09:47:26.062275+0000","last_clean":"2024-09-21T09:47:26.062275+0000","last_became_active":"2024-09-21T09:47:24.056915+0000","last_became_peered":"2024-09-21T09:47:24.056915+0000","last_unstale":"2024-09-21T09:47:26.062275+0000","last_undegraded":"2024-09-21T09:47:26.062275+0000","last_fullsized":"2024-09-21T09:47:26.062275+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061489+0000","last_change":"2024-09-21T09:47:24.057310+0000","last_active":"2024-09-21T09:47:26.061489+0000","last_peered":"2024-09-21T09:47:26.061489+0000","last_clean":"2024-09-21T09:47:26.061489+0000","last_became_active":"2024-09-21T09:47:24.057047+0000","last_became_peered":"2024-09-21T09:47:24.057047+0000","last_unstale":"2024-09-21T09:47:26.061489+0000","last_undegraded":"2024-09-21T09:47:26.061489+0000","last_fullsized":"2024-09-21T09:47:26.061489+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.283112+0000","last_change":"2024-09-21T09:47:24.060976+0000","last_active":"2024-09-21T09:47:25.283112+0000","last_peered":"2024-09-21T09:47:25.283112+0000","last_clean":"2024-09-21T09:47:25.283112+0000","last_became_active":"2024-09-21T09:47:24.060725+0000","last_became_peered":"2024-09-21T09:47:24.060725+0000","last_unstale":"2024-09-21T09:47:25.283112+0000","last_undegraded":"2024-09-21T09:47:25.283112+0000","last_fullsized":"2024-09-21T09:47:25.283112+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.059810+0000","last_change":"2024-09-21T09:47:24.053563+0000","last_active":"2024-09-21T09:47:26.059810+0000","last_peered":"2024-09-21T09:47:26.059810+0000","last_clean":"2024-09-21T09:47:26.059810+0000","last_became_active":"2024-09-21T09:47:24.052853+0000","last_became_peered":"2024-09-21T09:47:24.052853+0000","last_unstale":"2024-09-21T09:47:26.059810+0000","last_undegraded":"2024-09-21T09:47:26.059810+0000","last_fullsized":"2024-09-21T09:47:26.059810+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060904+0000","last_change":"2024-09-21T09:47:24.068204+0000","last_active":"2024-09-21T09:47:26.060904+0000","last_peered":"2024-09-21T09:47:26.060904+0000","last_clean":"2024-09-21T09:47:26.060904+0000","last_became_active":"2024-09-21T09:47:24.067911+0000","last_became_peered":"2024-09-21T09:47:24.067911+0000","last_unstale":"2024-09-21T09:47:26.060904+0000","last_undegraded":"2024-09-21T09:47:26.060904+0000","last_fullsized":"2024-09-21T09:47:26.060904+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475415+0000","last_change":"2024-09-21T09:47:24.058882+0000","last_active":"2024-09-21T09:47:45.475415+0000","last_peered":"2024-09-21T09:47:45.475415+0000","last_clean":"2024-09-21T09:47:45.475415+0000","last_became_active":"2024-09-21T09:47:24.058587+0000","last_became_peered":"2024-09-21T09:47:24.058587+0000","last_unstale":"2024-09-21T09:47:45.475415+0000","last_undegraded":"2024-09-21T09:47:45.475415+0000","last_fullsized":"2024-09-21T09:47:45.475415+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469392+0000","last_change":"2024-09-21T09:47:24.060446+0000","last_active":"2024-09-21T09:47:45.469392+0000","last_peered":"2024-09-21T09:47:45.469392+0000","last_clean":"2024-09-21T09:47:45.469392+0000","last_became_active":"2024-09-21T09:47:24.060165+0000","last_became_peered":"2024-09-21T09:47:24.060165+0000","last_unstale":"2024-09-21T09:47:45.469392+0000","last_undegraded":"2024-09-21T09:47:45.469392+0000","last_fullsized":"2024-09-21T09:47:45.469392+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.057761+0000","last_change":"2024-09-21T09:47:24.059974+0000","last_active":"2024-09-21T09:47:26.057761+0000","last_peered":"2024-09-21T09:47:26.057761+0000","last_clean":"2024-09-21T09:47:26.057761+0000","last_became_active":"2024-09-21T09:47:24.059270+0000","last_became_peered":"2024-09-21T09:47:24.059270+0000","last_unstale":"2024-09-21T09:47:26.057761+0000","last_undegraded":"2024-09-21T09:47:26.057761+0000","last_fullsized":"2024-09-21T09:47:26.057761+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470251+0000","last_change":"2024-09-21T09:47:24.048941+0000","last_active":"2024-09-21T09:47:45.470251+0000","last_peered":"2024-09-21T09:47:45.470251+0000","last_clean":"2024-09-21T09:47:45.470251+0000","last_became_active":"2024-09-21T09:47:24.048631+0000","last_became_peered":"2024-09-21T09:47:24.048631+0000","last_unstale":"2024-09-21T09:47:45.470251+0000","last_undegraded":"2024-09-21T09:47:45.470251+0000","last_fullsized":"2024-09-21T09:47:45.470251+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":16,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.064507+0000","last_change":"2024-09-21T09:47:20.866010+0000","last_active":"2024-09-21T09:47:26.064507+0000","last_peered":"2024-09-21T09:47:26.064507+0000","last_clean":"2024-09-21T09:47:26.064507+0000","last_became_active":"2024-09-21T09:47:20.865732+0000","last_became_peered":"2024-09-21T09:47:20.865732+0000","last_unstale":"2024-09-21T09:47:26.064507+0000","last_undegraded":"2024-09-21T09:47:26.064507+0000","last_fullsized":"2024-09-21T09:47:26.064507+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060967+0000","last_change":"2024-09-21T09:47:24.067813+0000","last_active":"2024-09-21T09:47:26.060967+0000","last_peered":"2024-09-21T09:47:26.060967+0000","last_clean":"2024-09-21T09:47:26.060967+0000","last_became_active":"2024-09-21T09:47:24.067412+0000","last_became_peered":"2024-09-21T09:47:24.067412+0000","last_unstale":"2024-09-21T09:47:26.060967+0000","last_undegraded":"2024-09-21T09:47:26.060967+0000","last_fullsized":"2024-09-21T09:47:26.060967+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":18,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-21T09:47:45.473703+0000","last_change":"2024-09-21T09:47:45.473958+0000","last_active":"2024-09-21T09:47:45.473671+0000","last_peered":"2024-09-21T09:47:26.064536+0000","last_clean":"2024-09-21T09:47:26.064536+0000","last_became_active":"2024-09-21T09:47:20.861410+0000","last_became_peered":"2024-09-21T09:47:20.861410+0000","last_unstale":"2024-09-21T09:47:45.473703+0000","last_undegraded":"2024-09-21T09:47:45.473703+0000","last_fullsized":"2024-09-21T09:47:45.473703+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:00:57.766502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061989+0000","last_change":"2024-09-21T09:47:20.859829+0000","last_active":"2024-09-21T09:47:26.061989+0000","last_peered":"2024-09-21T09:47:26.061989+0000","last_clean":"2024-09-21T09:47:26.061989+0000","last_became_active":"2024-09-21T09:47:20.859507+0000","last_became_peered":"2024-09-21T09:47:20.859507+0000","last_unstale":"2024-09-21T09:47:26.061989+0000","last_undegraded":"2024-09-21T09:47:26.061989+0000","last_fullsized":"2024-09-21T09:47:26.061989+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062234+0000","last_change":"2024-09-21T09:47:24.053702+0000","last_active":"2024-09-21T09:47:26.062234+0000","last_peered":"2024-09-21T09:47:26.062234+0000","last_clean":"2024-09-21T09:47:26.062234+0000","last_became_active":"2024-09-21T09:47:24.053295+0000","last_became_peered":"2024-09-21T09:47:24.053295+0000","last_unstale":"2024-09-21T09:47:26.062234+0000","last_undegraded":"2024-09-21T09:47:26.062234+0000","last_fullsized":"2024-09-21T09:47:26.062234+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475673+0000","last_change":"2024-09-21T09:47:20.863181+0000","last_active":"2024-09-21T09:47:45.475673+0000","last_peered":"2024-09-21T09:47:45.475673+0000","last_clean":"2024-09-21T09:47:45.475673+0000","last_became_active":"2024-09-21T09:47:20.862895+0000","last_became_peered":"2024-09-21T09:47:20.862895+0000","last_unstale":"2024-09-21T09:47:45.475673+0000","last_undegraded":"2024-09-21T09:47:45.475673+0000","last_fullsized":"2024-09-21T09:47:45.475673+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282439+0000","last_change":"2024-09-21T09:47:24.051248+0000","last_active":"2024-09-21T09:47:25.282439+0000","last_peered":"2024-09-21T09:47:25.282439+0000","last_clean":"2024-09-21T09:47:25.282439+0000","last_became_active":"2024-09-21T09:47:24.050918+0000","last_became_peered":"2024-09-21T09:47:24.050918+0000","last_unstale":"2024-09-21T09:47:25.282439+0000","last_undegraded":"2024-09-21T09:47:25.282439+0000","last_fullsized":"2024-09-21T09:47:25.282439+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061495+0000","last_change":"2024-09-21T09:47:20.865532+0000","last_active":"2024-09-21T09:47:26.061495+0000","last_peered":"2024-09-21T09:47:26.061495+0000","last_clean":"2024-09-21T09:47:26.061495+0000","last_became_active":"2024-09-21T09:47:20.865103+0000","last_became_peered":"2024-09-21T09:47:20.865103+0000","last_unstale":"2024-09-21T09:47:26.061495+0000","last_undegraded":"2024-09-21T09:47:26.061495+0000","last_fullsized":"2024-09-21T09:47:26.061495+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474292+0000","last_change":"2024-09-21T09:47:24.067330+0000","last_active":"2024-09-21T09:47:45.474292+0000","last_peered":"2024-09-21T09:47:45.474292+0000","last_clean":"2024-09-21T09:47:45.474292+0000","last_became_active":"2024-09-21T09:47:24.066939+0000","last_became_peered":"2024-09-21T09:47:24.066939+0000","last_unstale":"2024-09-21T09:47:45.474292+0000","last_undegraded":"2024-09-21T09:47:45.474292+0000","last_fullsized":"2024-09-21T09:47:45.474292+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474434+0000","last_change":"2024-09-21T09:47:20.860605+0000","last_active":"2024-09-21T09:47:45.474434+0000","last_peered":"2024-09-21T09:47:45.474434+0000","last_clean":"2024-09-21T09:47:45.474434+0000","last_became_active":"2024-09-21T09:47:20.860288+0000","last_became_peered":"2024-09-21T09:47:20.860288+0000","last_unstale":"2024-09-21T09:47:45.474434+0000","last_undegraded":"2024-09-21T09:47:45.474434+0000","last_fullsized":"2024-09-21T09:47:45.474434+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061722+0000","last_change":"2024-09-21T09:47:24.057860+0000","last_active":"2024-09-21T09:47:26.061722+0000","last_peered":"2024-09-21T09:47:26.061722+0000","last_clean":"2024-09-21T09:47:26.061722+0000","last_became_active":"2024-09-21T09:47:24.057594+0000","last_became_peered":"2024-09-21T09:47:24.057594+0000","last_unstale":"2024-09-21T09:47:26.061722+0000","last_undegraded":"2024-09-21T09:47:26.061722+0000","last_fullsized":"2024-09-21T09:47:26.061722+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.063141+0000","last_change":"2024-09-21T09:47:20.857026+0000","last_active":"2024-09-21T09:47:26.063141+0000","last_peered":"2024-09-21T09:47:26.063141+0000","last_clean":"2024-09-21T09:47:26.063141+0000","last_became_active":"2024-09-21T09:47:20.856719+0000","last_became_peered":"2024-09-21T09:47:20.856719+0000","last_unstale":"2024-09-21T09:47:26.063141+0000","last_undegraded":"2024-09-21T09:47:26.063141+0000","last_fullsized":"2024-09-21T09:47:26.063141+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471636+0000","last_change":"2024-09-21T09:47:24.059537+0000","last_active":"2024-09-21T09:47:45.471636+0000","last_peered":"2024-09-21T09:47:45.471636+0000","last_clean":"2024-09-21T09:47:45.471636+0000","last_became_active":"2024-09-21T09:47:24.059264+0000","last_became_peered":"2024-09-21T09:47:24.059264+0000","last_unstale":"2024-09-21T09:47:45.471636+0000","last_undegraded":"2024-09-21T09:47:45.471636+0000","last_fullsized":"2024-09-21T09:47:45.471636+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":21,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.066221+0000","last_change":"2024-09-21T09:47:20.864446+0000","last_active":"2024-09-21T09:47:26.066221+0000","last_peered":"2024-09-21T09:47:26.066221+0000","last_clean":"2024-09-21T09:47:26.066221+0000","last_became_active":"2024-09-21T09:47:20.864160+0000","last_became_peered":"2024-09-21T09:47:20.864160+0000","last_unstale":"2024-09-21T09:47:26.066221+0000","last_undegraded":"2024-09-21T09:47:26.066221+0000","last_fullsized":"2024-09-21T09:47:26.066221+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475529+0000","last_change":"2024-09-21T09:47:24.070010+0000","last_active":"2024-09-21T09:47:45.475529+0000","last_peered":"2024-09-21T09:47:45.475529+0000","last_clean":"2024-09-21T09:47:45.475529+0000","last_became_active":"2024-09-21T09:47:24.069698+0000","last_became_peered":"2024-09-21T09:47:24.069698+0000","last_unstale":"2024-09-21T09:47:45.475529+0000","last_undegraded":"2024-09-21T09:47:45.475529+0000","last_fullsized":"2024-09-21T09:47:45.475529+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.064742+0000","last_change":"2024-09-21T09:47:20.858242+0000","last_active":"2024-09-21T09:47:26.064742+0000","last_peered":"2024-09-21T09:47:26.064742+0000","last_clean":"2024-09-21T09:47:26.064742+0000","last_became_active":"2024-09-21T09:47:20.857951+0000","last_became_peered":"2024-09-21T09:47:20.857951+0000","last_unstale":"2024-09-21T09:47:26.064742+0000","last_undegraded":"2024-09-21T09:47:26.064742+0000","last_fullsized":"2024-09-21T09:47:26.064742+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060336+0000","last_change":"2024-09-21T09:47:24.063666+0000","last_active":"2024-09-21T09:47:26.060336+0000","last_peered":"2024-09-21T09:47:26.060336+0000","last_clean":"2024-09-21T09:47:26.060336+0000","last_became_active":"2024-09-21T09:47:24.063116+0000","last_became_peered":"2024-09-21T09:47:24.063116+0000","last_unstale":"2024-09-21T09:47:26.060336+0000","last_undegraded":"2024-09-21T09:47:26.060336+0000","last_fullsized":"2024-09-21T09:47:26.060336+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475534+0000","last_change":"2024-09-21T09:47:20.867134+0000","last_active":"2024-09-21T09:47:45.475534+0000","last_peered":"2024-09-21T09:47:45.475534+0000","last_clean":"2024-09-21T09:47:45.475534+0000","last_became_active":"2024-09-21T09:47:20.866875+0000","last_became_peered":"2024-09-21T09:47:20.866875+0000","last_unstale":"2024-09-21T09:47:45.475534+0000","last_undegraded":"2024-09-21T09:47:45.475534+0000","last_fullsized":"2024-09-21T09:47:45.475534+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475144+0000","last_change":"2024-09-21T09:47:24.069831+0000","last_active":"2024-09-21T09:47:45.475144+0000","last_peered":"2024-09-21T09:47:45.475144+0000","last_clean":"2024-09-21T09:47:45.475144+0000","last_became_active":"2024-09-21T09:47:24.069569+0000","last_became_peered":"2024-09-21T09:47:24.069569+0000","last_unstale":"2024-09-21T09:47:45.475144+0000","last_undegraded":"2024-09-21T09:47:45.475144+0000","last_fullsized":"2024-09-21T09:47:45.475144+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.058185+0000","last_change":"2024-09-21T09:47:20.862992+0000","last_active":"2024-09-21T09:47:26.058185+0000","last_peered":"2024-09-21T09:47:26.058185+0000","last_clean":"2024-09-21T09:47:26.058185+0000","last_became_active":"2024-09-21T09:47:20.862582+0000","last_became_peered":"2024-09-21T09:47:20.862582+0000","last_unstale":"2024-09-21T09:47:26.058185+0000","last_undegraded":"2024-09-21T09:47:26.058185+0000","last_fullsized":"2024-09-21T09:47:26.058185+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.057856+0000","last_change":"2024-09-21T09:47:24.058311+0000","last_active":"2024-09-21T09:47:26.057856+0000","last_peered":"2024-09-21T09:47:26.057856+0000","last_clean":"2024-09-21T09:47:26.057856+0000","last_became_active":"2024-09-21T09:47:24.057961+0000","last_became_peered":"2024-09-21T09:47:24.057961+0000","last_unstale":"2024-09-21T09:47:26.057856+0000","last_undegraded":"2024-09-21T09:47:26.057856+0000","last_fullsized":"2024-09-21T09:47:26.057856+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469194+0000","last_change":"2024-09-21T09:47:20.856509+0000","last_active":"2024-09-21T09:47:45.469194+0000","last_peered":"2024-09-21T09:47:45.469194+0000","last_clean":"2024-09-21T09:47:45.469194+0000","last_became_active":"2024-09-21T09:47:20.856229+0000","last_became_peered":"2024-09-21T09:47:20.856229+0000","last_unstale":"2024-09-21T09:47:45.469194+0000","last_undegraded":"2024-09-21T09:47:45.469194+0000","last_fullsized":"2024-09-21T09:47:45.469194+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474200+0000","last_change":"2024-09-21T09:47:24.069301+0000","last_active":"2024-09-21T09:47:45.474200+0000","last_peered":"2024-09-21T09:47:45.474200+0000","last_clean":"2024-09-21T09:47:45.474200+0000","last_became_active":"2024-09-21T09:47:24.068917+0000","last_became_peered":"2024-09-21T09:47:24.068917+0000","last_unstale":"2024-09-21T09:47:45.474200+0000","last_undegraded":"2024-09-21T09:47:45.474200+0000","last_fullsized":"2024-09-21T09:47:45.474200+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471246+0000","last_change":"2024-09-21T09:47:20.862118+0000","last_active":"2024-09-21T09:47:45.471246+0000","last_peered":"2024-09-21T09:47:45.471246+0000","last_clean":"2024-09-21T09:47:45.471246+0000","last_became_active":"2024-09-21T09:47:20.861815+0000","last_became_peered":"2024-09-21T09:47:20.861815+0000","last_unstale":"2024-09-21T09:47:45.471246+0000","last_undegraded":"2024-09-21T09:47:45.471246+0000","last_fullsized":"2024-09-21T09:47:45.471246+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474780+0000","last_change":"2024-09-21T09:47:24.069502+0000","last_active":"2024-09-21T09:47:45.474780+0000","last_peered":"2024-09-21T09:47:45.474780+0000","last_clean":"2024-09-21T09:47:45.474780+0000","last_became_active":"2024-09-21T09:47:24.069235+0000","last_became_peered":"2024-09-21T09:47:24.069235+0000","last_unstale":"2024-09-21T09:47:45.474780+0000","last_undegraded":"2024-09-21T09:47:45.474780+0000","last_fullsized":"2024-09-21T09:47:45.474780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469921+0000","last_change":"2024-09-21T09:47:20.862503+0000","last_active":"2024-09-21T09:47:45.469921+0000","last_peered":"2024-09-21T09:47:45.469921+0000","last_clean":"2024-09-21T09:47:45.469921+0000","last_became_active":"2024-09-21T09:47:20.862237+0000","last_became_peered":"2024-09-21T09:47:20.862237+0000","last_unstale":"2024-09-21T09:47:45.469921+0000","last_undegraded":"2024-09-21T09:47:45.469921+0000","last_fullsized":"2024-09-21T09:47:45.469921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476257+0000","last_change":"2024-09-21T09:47:24.069273+0000","last_active":"2024-09-21T09:47:45.476257+0000","last_peered":"2024-09-21T09:47:45.476257+0000","last_clean":"2024-09-21T09:47:45.476257+0000","last_became_active":"2024-09-21T09:47:24.068990+0000","last_became_peered":"2024-09-21T09:47:24.068990+0000","last_unstale":"2024-09-21T09:47:45.476257+0000","last_undegraded":"2024-09-21T09:47:45.476257+0000","last_fullsized":"2024-09-21T09:47:45.476257+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.043966+0000","last_change":"2024-09-21T09:47:20.858095+0000","last_active":"2024-09-21T09:47:26.043966+0000","last_peered":"2024-09-21T09:47:26.043966+0000","last_clean":"2024-09-21T09:47:26.043966+0000","last_became_active":"2024-09-21T09:47:20.857760+0000","last_became_peered":"2024-09-21T09:47:20.857760+0000","last_unstale":"2024-09-21T09:47:26.043966+0000","last_undegraded":"2024-09-21T09:47:26.043966+0000","last_fullsized":"2024-09-21T09:47:26.043966+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473915+0000","last_change":"2024-09-21T09:47:24.063342+0000","last_active":"2024-09-21T09:47:45.473915+0000","last_peered":"2024-09-21T09:47:45.473915+0000","last_clean":"2024-09-21T09:47:45.473915+0000","last_became_active":"2024-09-21T09:47:24.063070+0000","last_became_peered":"2024-09-21T09:47:24.063070+0000","last_unstale":"2024-09-21T09:47:45.473915+0000","last_undegraded":"2024-09-21T09:47:45.473915+0000","last_fullsized":"2024-09-21T09:47:45.473915+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469526+0000","last_change":"2024-09-21T09:47:20.859702+0000","last_active":"2024-09-21T09:47:45.469526+0000","last_peered":"2024-09-21T09:47:45.469526+0000","last_clean":"2024-09-21T09:47:45.469526+0000","last_became_active":"2024-09-21T09:47:20.859443+0000","last_became_peered":"2024-09-21T09:47:20.859443+0000","last_unstale":"2024-09-21T09:47:45.469526+0000","last_undegraded":"2024-09-21T09:47:45.469526+0000","last_fullsized":"2024-09-21T09:47:45.469526+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475617+0000","last_change":"2024-09-21T09:47:24.070338+0000","last_active":"2024-09-21T09:47:45.475617+0000","last_peered":"2024-09-21T09:47:45.475617+0000","last_clean":"2024-09-21T09:47:45.475617+0000","last_became_active":"2024-09-21T09:47:24.070074+0000","last_became_peered":"2024-09-21T09:47:24.070074+0000","last_unstale":"2024-09-21T09:47:45.475617+0000","last_undegraded":"2024-09-21T09:47:45.475617+0000","last_fullsized":"2024-09-21T09:47:45.475617+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":25,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473341+0000","last_change":"2024-09-21T09:46:57.426595+0000","last_active":"2024-09-21T09:47:45.473341+0000","last_peered":"2024-09-21T09:47:45.473341+0000","last_clean":"2024-09-21T09:47:45.473341+0000","last_became_active":"2024-09-21T09:46:57.418997+0000","last_became_peered":"2024-09-21T09:46:57.418997+0000","last_unstale":"2024-09-21T09:47:45.473341+0000","last_undegraded":"2024-09-21T09:47:45.473341+0000","last_fullsized":"2024-09-21T09:47:45.473341+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_clean_scrub_stamp":"2024-09-21T09:44:57.819716+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475502+0000","last_change":"2024-09-21T09:47:20.866500+0000","last_active":"2024-09-21T09:47:45.475502+0000","last_peered":"2024-09-21T09:47:45.475502+0000","last_clean":"2024-09-21T09:47:45.475502+0000","last_became_active":"2024-09-21T09:47:20.866224+0000","last_became_peered":"2024-09-21T09:47:20.866224+0000","last_unstale":"2024-09-21T09:47:45.475502+0000","last_undegraded":"2024-09-21T09:47:45.475502+0000","last_fullsized":"2024-09-21T09:47:45.475502+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060176+0000","last_change":"2024-09-21T09:47:24.061053+0000","last_active":"2024-09-21T09:47:26.060176+0000","last_peered":"2024-09-21T09:47:26.060176+0000","last_clean":"2024-09-21T09:47:26.060176+0000","last_became_active":"2024-09-21T09:47:24.060632+0000","last_became_peered":"2024-09-21T09:47:24.060632+0000","last_unstale":"2024-09-21T09:47:26.060176+0000","last_undegraded":"2024-09-21T09:47:26.060176+0000","last_fullsized":"2024-09-21T09:47:26.060176+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473227+0000","last_change":"2024-09-21T09:47:20.869192+0000","last_active":"2024-09-21T09:47:45.473227+0000","last_peered":"2024-09-21T09:47:45.473227+0000","last_clean":"2024-09-21T09:47:45.473227+0000","last_became_active":"2024-09-21T09:47:20.868800+0000","last_became_peered":"2024-09-21T09:47:20.868800+0000","last_unstale":"2024-09-21T09:47:45.473227+0000","last_undegraded":"2024-09-21T09:47:45.473227+0000","last_fullsized":"2024-09-21T09:47:45.473227+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.059857+0000","last_change":"2024-09-21T09:47:24.056658+0000","last_active":"2024-09-21T09:47:26.059857+0000","last_peered":"2024-09-21T09:47:26.059857+0000","last_clean":"2024-09-21T09:47:26.059857+0000","last_became_active":"2024-09-21T09:47:24.056366+0000","last_became_peered":"2024-09-21T09:47:24.056366+0000","last_unstale":"2024-09-21T09:47:26.059857+0000","last_undegraded":"2024-09-21T09:47:26.059857+0000","last_fullsized":"2024-09-21T09:47:26.059857+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469527+0000","last_change":"2024-09-21T09:47:20.861899+0000","last_active":"2024-09-21T09:47:45.469527+0000","last_peered":"2024-09-21T09:47:45.469527+0000","last_clean":"2024-09-21T09:47:45.469527+0000","last_became_active":"2024-09-21T09:47:20.861684+0000","last_became_peered":"2024-09-21T09:47:20.861684+0000","last_unstale":"2024-09-21T09:47:45.469527+0000","last_undegraded":"2024-09-21T09:47:45.469527+0000","last_fullsized":"2024-09-21T09:47:45.469527+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282439+0000","last_change":"2024-09-21T09:47:24.058235+0000","last_active":"2024-09-21T09:47:25.282439+0000","last_peered":"2024-09-21T09:47:25.282439+0000","last_clean":"2024-09-21T09:47:25.282439+0000","last_became_active":"2024-09-21T09:47:24.057972+0000","last_became_peered":"2024-09-21T09:47:24.057972+0000","last_unstale":"2024-09-21T09:47:25.282439+0000","last_undegraded":"2024-09-21T09:47:25.282439+0000","last_fullsized":"2024-09-21T09:47:25.282439+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475569+0000","last_change":"2024-09-21T09:47:20.861547+0000","last_active":"2024-09-21T09:47:45.475569+0000","last_peered":"2024-09-21T09:47:45.475569+0000","last_clean":"2024-09-21T09:47:45.475569+0000","last_became_active":"2024-09-21T09:47:20.861254+0000","last_became_peered":"2024-09-21T09:47:20.861254+0000","last_unstale":"2024-09-21T09:47:45.475569+0000","last_undegraded":"2024-09-21T09:47:45.475569+0000","last_fullsized":"2024-09-21T09:47:45.475569+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471048+0000","last_change":"2024-09-21T09:47:24.060191+0000","last_active":"2024-09-21T09:47:45.471048+0000","last_peered":"2024-09-21T09:47:45.471048+0000","last_clean":"2024-09-21T09:47:45.471048+0000","last_became_active":"2024-09-21T09:47:24.059929+0000","last_became_peered":"2024-09-21T09:47:24.059929+0000","last_unstale":"2024-09-21T09:47:45.471048+0000","last_undegraded":"2024-09-21T09:47:45.471048+0000","last_fullsized":"2024-09-21T09:47:45.471048+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282787+0000","last_change":"2024-09-21T09:47:20.862181+0000","last_active":"2024-09-21T09:47:25.282787+0000","last_peered":"2024-09-21T09:47:25.282787+0000","last_clean":"2024-09-21T09:47:25.282787+0000","last_became_active":"2024-09-21T09:47:20.861903+0000","last_became_peered":"2024-09-21T09:47:20.861903+0000","last_unstale":"2024-09-21T09:47:25.282787+0000","last_undegraded":"2024-09-21T09:47:25.282787+0000","last_fullsized":"2024-09-21T09:47:25.282787+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474023+0000","last_change":"2024-09-21T09:47:24.058849+0000","last_active":"2024-09-21T09:47:45.474023+0000","last_peered":"2024-09-21T09:47:45.474023+0000","last_clean":"2024-09-21T09:47:45.474023+0000","last_became_active":"2024-09-21T09:47:24.058564+0000","last_became_peered":"2024-09-21T09:47:24.058564+0000","last_unstale":"2024-09-21T09:47:45.474023+0000","last_undegraded":"2024-09-21T09:47:45.474023+0000","last_fullsized":"2024-09-21T09:47:45.474023+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476182+0000","last_change":"2024-09-21T09:47:20.866885+0000","last_active":"2024-09-21T09:47:45.476182+0000","last_peered":"2024-09-21T09:47:45.476182+0000","last_clean":"2024-09-21T09:47:45.476182+0000","last_became_active":"2024-09-21T09:47:20.866436+0000","last_became_peered":"2024-09-21T09:47:20.866436+0000","last_unstale":"2024-09-21T09:47:45.476182+0000","last_undegraded":"2024-09-21T09:47:45.476182+0000","last_fullsized":"2024-09-21T09:47:45.476182+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282714+0000","last_change":"2024-09-21T09:47:24.060367+0000","last_active":"2024-09-21T09:47:25.282714+0000","last_peered":"2024-09-21T09:47:25.282714+0000","last_clean":"2024-09-21T09:47:25.282714+0000","last_became_active":"2024-09-21T09:47:24.060083+0000","last_became_peered":"2024-09-21T09:47:24.060083+0000","last_unstale":"2024-09-21T09:47:25.282714+0000","last_undegraded":"2024-09-21T09:47:25.282714+0000","last_fullsized":"2024-09-21T09:47:25.282714+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":15,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.044158+0000","last_change":"2024-09-21T09:47:20.858947+0000","last_active":"2024-09-21T09:47:26.044158+0000","last_peered":"2024-09-21T09:47:26.044158+0000","last_clean":"2024-09-21T09:47:26.044158+0000","last_became_active":"2024-09-21T09:47:20.858680+0000","last_became_peered":"2024-09-21T09:47:20.858680+0000","last_unstale":"2024-09-21T09:47:26.044158+0000","last_undegraded":"2024-09-21T09:47:26.044158+0000","last_fullsized":"2024-09-21T09:47:26.044158+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062409+0000","last_change":"2024-09-21T09:47:24.060498+0000","last_active":"2024-09-21T09:47:26.062409+0000","last_peered":"2024-09-21T09:47:26.062409+0000","last_clean":"2024-09-21T09:47:26.062409+0000","last_became_active":"2024-09-21T09:47:24.060179+0000","last_became_peered":"2024-09-21T09:47:24.060179+0000","last_unstale":"2024-09-21T09:47:26.062409+0000","last_undegraded":"2024-09-21T09:47:26.062409+0000","last_fullsized":"2024-09-21T09:47:26.062409+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":17,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.064735+0000","last_change":"2024-09-21T09:47:20.865797+0000","last_active":"2024-09-21T09:47:26.064735+0000","last_peered":"2024-09-21T09:47:26.064735+0000","last_clean":"2024-09-21T09:47:26.064735+0000","last_became_active":"2024-09-21T09:47:20.865466+0000","last_became_peered":"2024-09-21T09:47:20.865466+0000","last_unstale":"2024-09-21T09:47:26.064735+0000","last_undegraded":"2024-09-21T09:47:26.064735+0000","last_fullsized":"2024-09-21T09:47:26.064735+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060421+0000","last_change":"2024-09-21T09:47:24.058690+0000","last_active":"2024-09-21T09:47:26.060421+0000","last_peered":"2024-09-21T09:47:26.060421+0000","last_clean":"2024-09-21T09:47:26.060421+0000","last_became_active":"2024-09-21T09:47:24.058371+0000","last_became_peered":"2024-09-21T09:47:24.058371+0000","last_unstale":"2024-09-21T09:47:26.060421+0000","last_undegraded":"2024-09-21T09:47:26.060421+0000","last_fullsized":"2024-09-21T09:47:26.060421+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473115+0000","last_change":"2024-09-21T09:47:20.868994+0000","last_active":"2024-09-21T09:47:45.473115+0000","last_peered":"2024-09-21T09:47:45.473115+0000","last_clean":"2024-09-21T09:47:45.473115+0000","last_became_active":"2024-09-21T09:47:20.868686+0000","last_became_peered":"2024-09-21T09:47:20.868686+0000","last_unstale":"2024-09-21T09:47:45.473115+0000","last_undegraded":"2024-09-21T09:47:45.473115+0000","last_fullsized":"2024-09-21T09:47:45.473115+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470397+0000","last_change":"2024-09-21T09:47:24.055068+0000","last_active":"2024-09-21T09:47:45.470397+0000","last_peered":"2024-09-21T09:47:45.470397+0000","last_clean":"2024-09-21T09:47:45.470397+0000","last_became_active":"2024-09-21T09:47:24.054785+0000","last_became_peered":"2024-09-21T09:47:24.054785+0000","last_unstale":"2024-09-21T09:47:45.470397+0000","last_undegraded":"2024-09-21T09:47:45.470397+0000","last_fullsized":"2024-09-21T09:47:45.470397+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.282677+0000","last_change":"2024-09-21T09:47:20.862674+0000","last_active":"2024-09-21T09:47:25.282677+0000","last_peered":"2024-09-21T09:47:25.282677+0000","last_clean":"2024-09-21T09:47:25.282677+0000","last_became_active":"2024-09-21T09:47:20.861988+0000","last_became_peered":"2024-09-21T09:47:20.861988+0000","last_unstale":"2024-09-21T09:47:25.282677+0000","last_undegraded":"2024-09-21T09:47:25.282677+0000","last_fullsized":"2024-09-21T09:47:25.282677+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.062272+0000","last_change":"2024-09-21T09:47:24.058826+0000","last_active":"2024-09-21T09:47:26.062272+0000","last_peered":"2024-09-21T09:47:26.062272+0000","last_clean":"2024-09-21T09:47:26.062272+0000","last_became_active":"2024-09-21T09:47:24.058565+0000","last_became_peered":"2024-09-21T09:47:24.058565+0000","last_unstale":"2024-09-21T09:47:26.062272+0000","last_undegraded":"2024-09-21T09:47:26.062272+0000","last_fullsized":"2024-09-21T09:47:26.062272+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":14,"reported_epoch":56,"state":"active+clean","last_fresh":"2024-09-21T09:47:25.283010+0000","last_change":"2024-09-21T09:47:20.861142+0000","last_active":"2024-09-21T09:47:25.283010+0000","last_peered":"2024-09-21T09:47:25.283010+0000","last_clean":"2024-09-21T09:47:25.283010+0000","last_became_active":"2024-09-21T09:47:20.860916+0000","last_became_peered":"2024-09-21T09:47:20.860916+0000","last_unstale":"2024-09-21T09:47:25.283010+0000","last_undegraded":"2024-09-21T09:47:25.283010+0000","last_fullsized":"2024-09-21T09:47:25.283010+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":14,"reported_epoch":58,"state":"peering","last_fresh":"2024-09-21T09:47:45.470626+0000","last_change":"2024-09-21T09:47:45.470771+0000","last_active":"2024-09-21T09:47:45.470595+0000","last_peered":"2024-09-21T09:47:26.061291+0000","last_clean":"2024-09-21T09:47:26.061291+0000","last_became_active":"2024-09-21T09:47:24.053945+0000","last_became_peered":"2024-09-21T09:47:24.053945+0000","last_unstale":"2024-09-21T09:47:45.470626+0000","last_undegraded":"2024-09-21T09:47:45.470626+0000","last_fullsized":"2024-09-21T09:47:45.470626+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:49:50.863157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,4],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475414+0000","last_change":"2024-09-21T09:47:20.865234+0000","last_active":"2024-09-21T09:47:45.475414+0000","last_peered":"2024-09-21T09:47:45.475414+0000","last_clean":"2024-09-21T09:47:45.475414+0000","last_became_active":"2024-09-21T09:47:20.863790+0000","last_became_peered":"2024-09-21T09:47:20.863790+0000","last_unstale":"2024-09-21T09:47:45.475414+0000","last_undegraded":"2024-09-21T09:47:45.475414+0000","last_fullsized":"2024-09-21T09:47:45.475414+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.060712+0000","last_change":"2024-09-21T09:47:24.056216+0000","last_active":"2024-09-21T09:47:26.060712+0000","last_peered":"2024-09-21T09:47:26.060712+0000","last_clean":"2024-09-21T09:47:26.060712+0000","last_became_active":"2024-09-21T09:47:24.055928+0000","last_became_peered":"2024-09-21T09:47:24.055928+0000","last_unstale":"2024-09-21T09:47:26.060712+0000","last_undegraded":"2024-09-21T09:47:26.060712+0000","last_fullsized":"2024-09-21T09:47:26.060712+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473935+0000","last_change":"2024-09-21T09:47:24.063533+0000","last_active":"2024-09-21T09:47:45.473935+0000","last_peered":"2024-09-21T09:47:45.473935+0000","last_clean":"2024-09-21T09:47:45.473935+0000","last_became_active":"2024-09-21T09:47:24.063293+0000","last_became_peered":"2024-09-21T09:47:24.063293+0000","last_unstale":"2024-09-21T09:47:45.473935+0000","last_undegraded":"2024-09-21T09:47:45.473935+0000","last_fullsized":"2024-09-21T09:47:45.473935+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474478+0000","last_change":"2024-09-21T09:47:20.864537+0000","last_active":"2024-09-21T09:47:45.474478+0000","last_peered":"2024-09-21T09:47:45.474478+0000","last_clean":"2024-09-21T09:47:45.474478+0000","last_became_active":"2024-09-21T09:47:20.864212+0000","last_became_peered":"2024-09-21T09:47:20.864212+0000","last_unstale":"2024-09-21T09:47:45.474478+0000","last_undegraded":"2024-09-21T09:47:45.474478+0000","last_fullsized":"2024-09-21T09:47:45.474478+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774197+0000","last_change":"2024-09-21T09:47:46.774197+0000","last_active":"2024-09-21T09:47:46.774197+0000","last_peered":"2024-09-21T09:47:46.774197+0000","last_clean":"2024-09-21T09:47:46.774197+0000","last_became_active":"2024-09-21T09:47:46.773846+0000","last_became_peered":"2024-09-21T09:47:46.773846+0000","last_unstale":"2024-09-21T09:47:46.774197+0000","last_undegraded":"2024-09-21T09:47:46.774197+0000","last_fullsized":"2024-09-21T09:47:46.774197+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475812+0000","last_change":"2024-09-21T09:47:20.866709+0000","last_active":"2024-09-21T09:47:45.475812+0000","last_peered":"2024-09-21T09:47:45.475812+0000","last_clean":"2024-09-21T09:47:45.475812+0000","last_became_active":"2024-09-21T09:47:20.866026+0000","last_became_peered":"2024-09-21T09:47:20.866026+0000","last_unstale":"2024-09-21T09:47:45.475812+0000","last_undegraded":"2024-09-21T09:47:45.475812+0000","last_fullsized":"2024-09-21T09:47:45.475812+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.061633+0000","last_change":"2024-09-21T09:47:24.060801+0000","last_active":"2024-09-21T09:47:26.061633+0000","last_peered":"2024-09-21T09:47:26.061633+0000","last_clean":"2024-09-21T09:47:26.061633+0000","last_became_active":"2024-09-21T09:47:24.060536+0000","last_became_peered":"2024-09-21T09:47:24.060536+0000","last_unstale":"2024-09-21T09:47:26.061633+0000","last_undegraded":"2024-09-21T09:47:26.061633+0000","last_fullsized":"2024-09-21T09:47:26.061633+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":16,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.063670+0000","last_change":"2024-09-21T09:47:20.864529+0000","last_active":"2024-09-21T09:47:26.063670+0000","last_peered":"2024-09-21T09:47:26.063670+0000","last_clean":"2024-09-21T09:47:26.063670+0000","last_became_active":"2024-09-21T09:47:20.864185+0000","last_became_peered":"2024-09-21T09:47:20.864185+0000","last_unstale":"2024-09-21T09:47:26.063670+0000","last_undegraded":"2024-09-21T09:47:26.063670+0000","last_fullsized":"2024-09-21T09:47:26.063670+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":12,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.059837+0000","last_change":"2024-09-21T09:47:24.065677+0000","last_active":"2024-09-21T09:47:26.059837+0000","last_peered":"2024-09-21T09:47:26.059837+0000","last_clean":"2024-09-21T09:47:26.059837+0000","last_became_active":"2024-09-21T09:47:24.065373+0000","last_became_peered":"2024-09-21T09:47:24.065373+0000","last_unstale":"2024-09-21T09:47:26.059837+0000","last_undegraded":"2024-09-21T09:47:26.059837+0000","last_fullsized":"2024-09-21T09:47:26.059837+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":19,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-21T09:47:26.065338+0000","last_change":"2024-09-21T09:47:20.865947+0000","last_active":"2024-09-21T09:47:26.065338+0000","last_peered":"2024-09-21T09:47:26.065338+0000","last_clean":"2024-09-21T09:47:26.065338+0000","last_became_active":"2024-09-21T09:47:20.865683+0000","last_became_peered":"2024-09-21T09:47:20.865683+0000","last_unstale":"2024-09-21T09:47:26.065338+0000","last_undegraded":"2024-09-21T09:47:26.065338+0000","last_fullsized":"2024-09-21T09:47:26.065338+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430221,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1152,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1179648,"data_stored":835428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52600000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49399999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54400000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.505}]}]},{"osd":6,"up_from":42,"seq":180388626449,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298504,"kb_used_data":1148,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446648,"statfs":{"total":95995035648,"available":95689367552,"internally_reserved":0,"allocated":1175552,"data_stored":835829,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.57499999999999996}]},{"osd":1,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.63600000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.60799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64900000000000002}]}]},{"osd":5,"up_from":36,"seq":154618822678,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":589824,"data_stored":245795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.54600000000000004}]},{"osd":1,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.56200000000000006}]},{"osd":2,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.376}]},{"osd":4,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:47:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.57499999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61599999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018907,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":606208,"data_stored":251705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.56299999999999994,"15min":0.57299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68999999999999995,"5min":0.748,"15min":0.748},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.432,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.67400000000000004}]},{"osd":1,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.38,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.65600000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.435,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67700000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.71599999999999997,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.51900000000000002,"15min":0.51600000000000001},"min":{"1min":0.42399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.624,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.60499999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.51800000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.64200000000000002,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.44900000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.68300000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:47:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.56999999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.443}]}]},{"osd":3,"up_from":25,"seq":107374182432,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297932,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447220,"statfs":{"total":95995035648,"available":95689953280,"internally_reserved":0,"allocated":581632,"data_stored":239134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.39800000000000002,"15min":0.40000000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.55500000000000005,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.251},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41299999999999998,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.59499999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.34399999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44500000000000001,"15min":0.45000000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42799999999999999,"15min":0.43099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.69299999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65600000000000003,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:46:43 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.44600000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:47:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.498},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.45400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":244468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.35999999999999999,"15min":0.35399999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.246},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.27300000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.57799999999999996,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59999999999999998,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.442,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63800000000000001,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:47:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.46800000000000003,"15min":0.47499999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.56200000000000006,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63900000000000001,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.27100000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:47:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73099999999999998,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.75900000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.71099999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.627}]},{"osd":6,"last update":"Sat Sep 21 09:47:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.79200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.73799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":560,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":573440,"data_stored":237953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.34799999999999998,"15min":0.32400000000000001},"min":{"1min":0.24399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.52500000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.435},{"interface":"front","average":{"1min":0.379,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.28199999999999997,"5min":0.214,"15min":0.214},"max":{"1min":0.47899999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.29199999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.375,"15min":0.375},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.49399999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.41099999999999998,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.63500000000000001,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:47:22 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57299999999999995,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.65000000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71199999999999997,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.67600000000000005,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.70099999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.747}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72399999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738414,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298508,"kb_used_data":1144,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446644,"statfs":{"total":95995035648,"available":95689363456,"internally_reserved":0,"allocated":1171456,"data_stored":828562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:47:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.33800000000000002,"15min":0.316},"min":{"1min":0.24199999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.374,"15min":0.34999999999999998},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:47:01 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.27500000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.59099999999999997,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.39500000000000002,"15min":0.38500000000000001},"min":{"1min":0.33300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Sep 21 09:47:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.317},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.497},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.33100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:47:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.56000000000000005,"15min":0.55200000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.629,"5min":0.57799999999999996,"15min":0.56999999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.65300000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:47:06 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Sep 21 09:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.621}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:47:51.424 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:47:51.686 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:51 smithi033 ceph-mon[39045]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:51.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:51 smithi033 ceph-mon[27969]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:51 smithi105 ceph-mon[31286]: from='client.24407 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:52 smithi033 ceph-mon[27969]: pgmap v202: 97 pgs: 1 activating, 1 peering, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:52 smithi033 ceph-mon[39045]: pgmap v202: 97 pgs: 1 activating, 1 peering, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:52 smithi105 ceph-mon[31286]: pgmap v202: 97 pgs: 1 activating, 1 peering, 95 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:53.656 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:53.657 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-21T09:47:54.365 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":203,"stamp":"2024-09-21T09:47:53.384396+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":290,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385284,"kb_used_data":6424,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575932,"statfs":{"total":767960285184,"available":765517754368,"internally_reserved":0,"allocated":6578176,"data_stored":3784686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002196"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473431+0000","last_change":"2024-09-21T09:47:24.066961+0000","last_active":"2024-09-21T09:47:45.473431+0000","last_peered":"2024-09-21T09:47:45.473431+0000","last_clean":"2024-09-21T09:47:45.473431+0000","last_became_active":"2024-09-21T09:47:24.066672+0000","last_became_peered":"2024-09-21T09:47:24.066672+0000","last_unstale":"2024-09-21T09:47:45.473431+0000","last_undegraded":"2024-09-21T09:47:45.473431+0000","last_fullsized":"2024-09-21T09:47:45.473431+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475762+0000","last_change":"2024-09-21T09:47:24.059165+0000","last_active":"2024-09-21T09:47:45.475762+0000","last_peered":"2024-09-21T09:47:45.475762+0000","last_clean":"2024-09-21T09:47:45.475762+0000","last_became_active":"2024-09-21T09:47:24.058702+0000","last_became_peered":"2024-09-21T09:47:24.058702+0000","last_unstale":"2024-09-21T09:47:45.475762+0000","last_undegraded":"2024-09-21T09:47:45.475762+0000","last_fullsized":"2024-09-21T09:47:45.475762+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.472960+0000","last_change":"2024-09-21T09:47:24.060359+0000","last_active":"2024-09-21T09:47:45.472960+0000","last_peered":"2024-09-21T09:47:45.472960+0000","last_clean":"2024-09-21T09:47:45.472960+0000","last_became_active":"2024-09-21T09:47:24.060088+0000","last_became_peered":"2024-09-21T09:47:24.060088+0000","last_unstale":"2024-09-21T09:47:45.472960+0000","last_undegraded":"2024-09-21T09:47:45.472960+0000","last_fullsized":"2024-09-21T09:47:45.472960+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473799+0000","last_change":"2024-09-21T09:47:24.065047+0000","last_active":"2024-09-21T09:47:45.473799+0000","last_peered":"2024-09-21T09:47:45.473799+0000","last_clean":"2024-09-21T09:47:45.473799+0000","last_became_active":"2024-09-21T09:47:24.064743+0000","last_became_peered":"2024-09-21T09:47:24.064743+0000","last_unstale":"2024-09-21T09:47:45.473799+0000","last_undegraded":"2024-09-21T09:47:45.473799+0000","last_fullsized":"2024-09-21T09:47:45.473799+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475055+0000","last_change":"2024-09-21T09:47:24.069692+0000","last_active":"2024-09-21T09:47:45.475055+0000","last_peered":"2024-09-21T09:47:45.475055+0000","last_clean":"2024-09-21T09:47:45.475055+0000","last_became_active":"2024-09-21T09:47:24.069090+0000","last_became_peered":"2024-09-21T09:47:24.069090+0000","last_unstale":"2024-09-21T09:47:45.475055+0000","last_undegraded":"2024-09-21T09:47:45.475055+0000","last_fullsized":"2024-09-21T09:47:45.475055+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475319+0000","last_change":"2024-09-21T09:47:24.053540+0000","last_active":"2024-09-21T09:47:45.475319+0000","last_peered":"2024-09-21T09:47:45.475319+0000","last_clean":"2024-09-21T09:47:45.475319+0000","last_became_active":"2024-09-21T09:47:24.053163+0000","last_became_peered":"2024-09-21T09:47:24.053163+0000","last_unstale":"2024-09-21T09:47:45.475319+0000","last_undegraded":"2024-09-21T09:47:45.475319+0000","last_fullsized":"2024-09-21T09:47:45.475319+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774760+0000","last_change":"2024-09-21T09:47:46.774760+0000","last_active":"2024-09-21T09:47:46.774760+0000","last_peered":"2024-09-21T09:47:46.774760+0000","last_clean":"2024-09-21T09:47:46.774760+0000","last_became_active":"2024-09-21T09:47:46.774445+0000","last_became_peered":"2024-09-21T09:47:46.774445+0000","last_unstale":"2024-09-21T09:47:46.774760+0000","last_undegraded":"2024-09-21T09:47:46.774760+0000","last_fullsized":"2024-09-21T09:47:46.774760+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469675+0000","last_change":"2024-09-21T09:47:24.064091+0000","last_active":"2024-09-21T09:47:45.469675+0000","last_peered":"2024-09-21T09:47:45.469675+0000","last_clean":"2024-09-21T09:47:45.469675+0000","last_became_active":"2024-09-21T09:47:24.063698+0000","last_became_peered":"2024-09-21T09:47:24.063698+0000","last_unstale":"2024-09-21T09:47:45.469675+0000","last_undegraded":"2024-09-21T09:47:45.469675+0000","last_fullsized":"2024-09-21T09:47:45.469675+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476570+0000","last_change":"2024-09-21T09:47:24.071747+0000","last_active":"2024-09-21T09:47:45.476570+0000","last_peered":"2024-09-21T09:47:45.476570+0000","last_clean":"2024-09-21T09:47:45.476570+0000","last_became_active":"2024-09-21T09:47:24.071475+0000","last_became_peered":"2024-09-21T09:47:24.071475+0000","last_unstale":"2024-09-21T09:47:45.476570+0000","last_undegraded":"2024-09-21T09:47:45.476570+0000","last_fullsized":"2024-09-21T09:47:45.476570+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475068+0000","last_change":"2024-09-21T09:47:24.058216+0000","last_active":"2024-09-21T09:47:45.475068+0000","last_peered":"2024-09-21T09:47:45.475068+0000","last_clean":"2024-09-21T09:47:45.475068+0000","last_became_active":"2024-09-21T09:47:24.057940+0000","last_became_peered":"2024-09-21T09:47:24.057940+0000","last_unstale":"2024-09-21T09:47:45.475068+0000","last_undegraded":"2024-09-21T09:47:45.475068+0000","last_fullsized":"2024-09-21T09:47:45.475068+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474927+0000","last_change":"2024-09-21T09:47:24.053543+0000","last_active":"2024-09-21T09:47:45.474927+0000","last_peered":"2024-09-21T09:47:45.474927+0000","last_clean":"2024-09-21T09:47:45.474927+0000","last_became_active":"2024-09-21T09:47:24.053264+0000","last_became_peered":"2024-09-21T09:47:24.053264+0000","last_unstale":"2024-09-21T09:47:45.474927+0000","last_undegraded":"2024-09-21T09:47:45.474927+0000","last_fullsized":"2024-09-21T09:47:45.474927+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.850968+0000","last_change":"2024-09-21T09:47:46.850968+0000","last_active":"2024-09-21T09:47:46.850968+0000","last_peered":"2024-09-21T09:47:46.850968+0000","last_clean":"2024-09-21T09:47:46.850968+0000","last_became_active":"2024-09-21T09:47:46.850604+0000","last_became_peered":"2024-09-21T09:47:46.850604+0000","last_unstale":"2024-09-21T09:47:46.850968+0000","last_undegraded":"2024-09-21T09:47:46.850968+0000","last_fullsized":"2024-09-21T09:47:46.850968+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470142+0000","last_change":"2024-09-21T09:47:24.064334+0000","last_active":"2024-09-21T09:47:45.470142+0000","last_peered":"2024-09-21T09:47:45.470142+0000","last_clean":"2024-09-21T09:47:45.470142+0000","last_became_active":"2024-09-21T09:47:24.064063+0000","last_became_peered":"2024-09-21T09:47:24.064063+0000","last_unstale":"2024-09-21T09:47:45.470142+0000","last_undegraded":"2024-09-21T09:47:45.470142+0000","last_fullsized":"2024-09-21T09:47:45.470142+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473158+0000","last_change":"2024-09-21T09:47:24.058842+0000","last_active":"2024-09-21T09:47:45.473158+0000","last_peered":"2024-09-21T09:47:45.473158+0000","last_clean":"2024-09-21T09:47:45.473158+0000","last_became_active":"2024-09-21T09:47:24.058245+0000","last_became_peered":"2024-09-21T09:47:24.058245+0000","last_unstale":"2024-09-21T09:47:45.473158+0000","last_undegraded":"2024-09-21T09:47:45.473158+0000","last_fullsized":"2024-09-21T09:47:45.473158+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475221+0000","last_change":"2024-09-21T09:47:24.052028+0000","last_active":"2024-09-21T09:47:45.475221+0000","last_peered":"2024-09-21T09:47:45.475221+0000","last_clean":"2024-09-21T09:47:45.475221+0000","last_became_active":"2024-09-21T09:47:24.051766+0000","last_became_peered":"2024-09-21T09:47:24.051766+0000","last_unstale":"2024-09-21T09:47:45.475221+0000","last_undegraded":"2024-09-21T09:47:45.475221+0000","last_fullsized":"2024-09-21T09:47:45.475221+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473412+0000","last_change":"2024-09-21T09:47:24.060117+0000","last_active":"2024-09-21T09:47:45.473412+0000","last_peered":"2024-09-21T09:47:45.473412+0000","last_clean":"2024-09-21T09:47:45.473412+0000","last_became_active":"2024-09-21T09:47:24.059852+0000","last_became_peered":"2024-09-21T09:47:24.059852+0000","last_unstale":"2024-09-21T09:47:45.473412+0000","last_undegraded":"2024-09-21T09:47:45.473412+0000","last_fullsized":"2024-09-21T09:47:45.473412+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.480400+0000","last_change":"2024-09-21T09:47:46.480400+0000","last_active":"2024-09-21T09:47:46.480400+0000","last_peered":"2024-09-21T09:47:46.480400+0000","last_clean":"2024-09-21T09:47:46.480400+0000","last_became_active":"2024-09-21T09:47:46.480103+0000","last_became_peered":"2024-09-21T09:47:46.480103+0000","last_unstale":"2024-09-21T09:47:46.480400+0000","last_undegraded":"2024-09-21T09:47:46.480400+0000","last_fullsized":"2024-09-21T09:47:46.480400+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469780+0000","last_change":"2024-09-21T09:47:24.057147+0000","last_active":"2024-09-21T09:47:45.469780+0000","last_peered":"2024-09-21T09:47:45.469780+0000","last_clean":"2024-09-21T09:47:45.469780+0000","last_became_active":"2024-09-21T09:47:24.056798+0000","last_became_peered":"2024-09-21T09:47:24.056798+0000","last_unstale":"2024-09-21T09:47:45.469780+0000","last_undegraded":"2024-09-21T09:47:45.469780+0000","last_fullsized":"2024-09-21T09:47:45.469780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475925+0000","last_change":"2024-09-21T09:47:24.070640+0000","last_active":"2024-09-21T09:47:45.475925+0000","last_peered":"2024-09-21T09:47:45.475925+0000","last_clean":"2024-09-21T09:47:45.475925+0000","last_became_active":"2024-09-21T09:47:24.070359+0000","last_became_peered":"2024-09-21T09:47:24.070359+0000","last_unstale":"2024-09-21T09:47:45.475925+0000","last_undegraded":"2024-09-21T09:47:45.475925+0000","last_fullsized":"2024-09-21T09:47:45.475925+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474605+0000","last_change":"2024-09-21T09:47:24.056659+0000","last_active":"2024-09-21T09:47:45.474605+0000","last_peered":"2024-09-21T09:47:45.474605+0000","last_clean":"2024-09-21T09:47:45.474605+0000","last_became_active":"2024-09-21T09:47:24.056360+0000","last_became_peered":"2024-09-21T09:47:24.056360+0000","last_unstale":"2024-09-21T09:47:45.474605+0000","last_undegraded":"2024-09-21T09:47:45.474605+0000","last_fullsized":"2024-09-21T09:47:45.474605+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473450+0000","last_change":"2024-09-21T09:47:24.065154+0000","last_active":"2024-09-21T09:47:45.473450+0000","last_peered":"2024-09-21T09:47:45.473450+0000","last_clean":"2024-09-21T09:47:45.473450+0000","last_became_active":"2024-09-21T09:47:24.064231+0000","last_became_peered":"2024-09-21T09:47:24.064231+0000","last_unstale":"2024-09-21T09:47:45.473450+0000","last_undegraded":"2024-09-21T09:47:45.473450+0000","last_fullsized":"2024-09-21T09:47:45.473450+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475326+0000","last_change":"2024-09-21T09:47:24.062716+0000","last_active":"2024-09-21T09:47:45.475326+0000","last_peered":"2024-09-21T09:47:45.475326+0000","last_clean":"2024-09-21T09:47:45.475326+0000","last_became_active":"2024-09-21T09:47:24.062437+0000","last_became_peered":"2024-09-21T09:47:24.062437+0000","last_unstale":"2024-09-21T09:47:45.475326+0000","last_undegraded":"2024-09-21T09:47:45.475326+0000","last_fullsized":"2024-09-21T09:47:45.475326+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470046+0000","last_change":"2024-09-21T09:47:24.051387+0000","last_active":"2024-09-21T09:47:45.470046+0000","last_peered":"2024-09-21T09:47:45.470046+0000","last_clean":"2024-09-21T09:47:45.470046+0000","last_became_active":"2024-09-21T09:47:24.051090+0000","last_became_peered":"2024-09-21T09:47:24.051090+0000","last_unstale":"2024-09-21T09:47:45.470046+0000","last_undegraded":"2024-09-21T09:47:45.470046+0000","last_fullsized":"2024-09-21T09:47:45.470046+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473500+0000","last_change":"2024-09-21T09:47:24.058079+0000","last_active":"2024-09-21T09:47:45.473500+0000","last_peered":"2024-09-21T09:47:45.473500+0000","last_clean":"2024-09-21T09:47:45.473500+0000","last_became_active":"2024-09-21T09:47:24.057753+0000","last_became_peered":"2024-09-21T09:47:24.057753+0000","last_unstale":"2024-09-21T09:47:45.473500+0000","last_undegraded":"2024-09-21T09:47:45.473500+0000","last_fullsized":"2024-09-21T09:47:45.473500+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469933+0000","last_change":"2024-09-21T09:47:24.057220+0000","last_active":"2024-09-21T09:47:45.469933+0000","last_peered":"2024-09-21T09:47:45.469933+0000","last_clean":"2024-09-21T09:47:45.469933+0000","last_became_active":"2024-09-21T09:47:24.056915+0000","last_became_peered":"2024-09-21T09:47:24.056915+0000","last_unstale":"2024-09-21T09:47:45.469933+0000","last_undegraded":"2024-09-21T09:47:45.469933+0000","last_fullsized":"2024-09-21T09:47:45.469933+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475187+0000","last_change":"2024-09-21T09:47:24.057310+0000","last_active":"2024-09-21T09:47:45.475187+0000","last_peered":"2024-09-21T09:47:45.475187+0000","last_clean":"2024-09-21T09:47:45.475187+0000","last_became_active":"2024-09-21T09:47:24.057047+0000","last_became_peered":"2024-09-21T09:47:24.057047+0000","last_unstale":"2024-09-21T09:47:45.475187+0000","last_undegraded":"2024-09-21T09:47:45.475187+0000","last_fullsized":"2024-09-21T09:47:45.475187+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475460+0000","last_change":"2024-09-21T09:47:24.060976+0000","last_active":"2024-09-21T09:47:45.475460+0000","last_peered":"2024-09-21T09:47:45.475460+0000","last_clean":"2024-09-21T09:47:45.475460+0000","last_became_active":"2024-09-21T09:47:24.060725+0000","last_became_peered":"2024-09-21T09:47:24.060725+0000","last_unstale":"2024-09-21T09:47:45.475460+0000","last_undegraded":"2024-09-21T09:47:45.475460+0000","last_fullsized":"2024-09-21T09:47:45.475460+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470271+0000","last_change":"2024-09-21T09:47:24.053563+0000","last_active":"2024-09-21T09:47:45.470271+0000","last_peered":"2024-09-21T09:47:45.470271+0000","last_clean":"2024-09-21T09:47:45.470271+0000","last_became_active":"2024-09-21T09:47:24.052853+0000","last_became_peered":"2024-09-21T09:47:24.052853+0000","last_unstale":"2024-09-21T09:47:45.470271+0000","last_undegraded":"2024-09-21T09:47:45.470271+0000","last_fullsized":"2024-09-21T09:47:45.470271+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473532+0000","last_change":"2024-09-21T09:47:24.068204+0000","last_active":"2024-09-21T09:47:45.473532+0000","last_peered":"2024-09-21T09:47:45.473532+0000","last_clean":"2024-09-21T09:47:45.473532+0000","last_became_active":"2024-09-21T09:47:24.067911+0000","last_became_peered":"2024-09-21T09:47:24.067911+0000","last_unstale":"2024-09-21T09:47:45.473532+0000","last_undegraded":"2024-09-21T09:47:45.473532+0000","last_fullsized":"2024-09-21T09:47:45.473532+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475415+0000","last_change":"2024-09-21T09:47:24.058882+0000","last_active":"2024-09-21T09:47:45.475415+0000","last_peered":"2024-09-21T09:47:45.475415+0000","last_clean":"2024-09-21T09:47:45.475415+0000","last_became_active":"2024-09-21T09:47:24.058587+0000","last_became_peered":"2024-09-21T09:47:24.058587+0000","last_unstale":"2024-09-21T09:47:45.475415+0000","last_undegraded":"2024-09-21T09:47:45.475415+0000","last_fullsized":"2024-09-21T09:47:45.475415+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469392+0000","last_change":"2024-09-21T09:47:24.060446+0000","last_active":"2024-09-21T09:47:45.469392+0000","last_peered":"2024-09-21T09:47:45.469392+0000","last_clean":"2024-09-21T09:47:45.469392+0000","last_became_active":"2024-09-21T09:47:24.060165+0000","last_became_peered":"2024-09-21T09:47:24.060165+0000","last_unstale":"2024-09-21T09:47:45.469392+0000","last_undegraded":"2024-09-21T09:47:45.469392+0000","last_fullsized":"2024-09-21T09:47:45.469392+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473573+0000","last_change":"2024-09-21T09:47:24.059974+0000","last_active":"2024-09-21T09:47:45.473573+0000","last_peered":"2024-09-21T09:47:45.473573+0000","last_clean":"2024-09-21T09:47:45.473573+0000","last_became_active":"2024-09-21T09:47:24.059270+0000","last_became_peered":"2024-09-21T09:47:24.059270+0000","last_unstale":"2024-09-21T09:47:45.473573+0000","last_undegraded":"2024-09-21T09:47:45.473573+0000","last_fullsized":"2024-09-21T09:47:45.473573+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470251+0000","last_change":"2024-09-21T09:47:24.048941+0000","last_active":"2024-09-21T09:47:45.470251+0000","last_peered":"2024-09-21T09:47:45.470251+0000","last_clean":"2024-09-21T09:47:45.470251+0000","last_became_active":"2024-09-21T09:47:24.048631+0000","last_became_peered":"2024-09-21T09:47:24.048631+0000","last_unstale":"2024-09-21T09:47:45.470251+0000","last_undegraded":"2024-09-21T09:47:45.470251+0000","last_fullsized":"2024-09-21T09:47:45.470251+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470304+0000","last_change":"2024-09-21T09:47:20.866010+0000","last_active":"2024-09-21T09:47:45.470304+0000","last_peered":"2024-09-21T09:47:45.470304+0000","last_clean":"2024-09-21T09:47:45.470304+0000","last_became_active":"2024-09-21T09:47:20.865732+0000","last_became_peered":"2024-09-21T09:47:20.865732+0000","last_unstale":"2024-09-21T09:47:45.470304+0000","last_undegraded":"2024-09-21T09:47:45.470304+0000","last_fullsized":"2024-09-21T09:47:45.470304+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473645+0000","last_change":"2024-09-21T09:47:24.067813+0000","last_active":"2024-09-21T09:47:45.473645+0000","last_peered":"2024-09-21T09:47:45.473645+0000","last_clean":"2024-09-21T09:47:45.473645+0000","last_became_active":"2024-09-21T09:47:24.067412+0000","last_became_peered":"2024-09-21T09:47:24.067412+0000","last_unstale":"2024-09-21T09:47:45.473645+0000","last_undegraded":"2024-09-21T09:47:45.473645+0000","last_fullsized":"2024-09-21T09:47:45.473645+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":27,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.481320+0000","last_change":"2024-09-21T09:47:46.481320+0000","last_active":"2024-09-21T09:47:46.481320+0000","last_peered":"2024-09-21T09:47:46.481320+0000","last_clean":"2024-09-21T09:47:46.481320+0000","last_became_active":"2024-09-21T09:47:46.480905+0000","last_became_peered":"2024-09-21T09:47:46.480905+0000","last_unstale":"2024-09-21T09:47:46.481320+0000","last_undegraded":"2024-09-21T09:47:46.481320+0000","last_fullsized":"2024-09-21T09:47:46.481320+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:23.831301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470210+0000","last_change":"2024-09-21T09:47:20.859829+0000","last_active":"2024-09-21T09:47:45.470210+0000","last_peered":"2024-09-21T09:47:45.470210+0000","last_clean":"2024-09-21T09:47:45.470210+0000","last_became_active":"2024-09-21T09:47:20.859507+0000","last_became_peered":"2024-09-21T09:47:20.859507+0000","last_unstale":"2024-09-21T09:47:45.470210+0000","last_undegraded":"2024-09-21T09:47:45.470210+0000","last_fullsized":"2024-09-21T09:47:45.470210+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469715+0000","last_change":"2024-09-21T09:47:24.053702+0000","last_active":"2024-09-21T09:47:45.469715+0000","last_peered":"2024-09-21T09:47:45.469715+0000","last_clean":"2024-09-21T09:47:45.469715+0000","last_became_active":"2024-09-21T09:47:24.053295+0000","last_became_peered":"2024-09-21T09:47:24.053295+0000","last_unstale":"2024-09-21T09:47:45.469715+0000","last_undegraded":"2024-09-21T09:47:45.469715+0000","last_fullsized":"2024-09-21T09:47:45.469715+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475673+0000","last_change":"2024-09-21T09:47:20.863181+0000","last_active":"2024-09-21T09:47:45.475673+0000","last_peered":"2024-09-21T09:47:45.475673+0000","last_clean":"2024-09-21T09:47:45.475673+0000","last_became_active":"2024-09-21T09:47:20.862895+0000","last_became_peered":"2024-09-21T09:47:20.862895+0000","last_unstale":"2024-09-21T09:47:45.475673+0000","last_undegraded":"2024-09-21T09:47:45.475673+0000","last_fullsized":"2024-09-21T09:47:45.475673+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475132+0000","last_change":"2024-09-21T09:47:24.051248+0000","last_active":"2024-09-21T09:47:45.475132+0000","last_peered":"2024-09-21T09:47:45.475132+0000","last_clean":"2024-09-21T09:47:45.475132+0000","last_became_active":"2024-09-21T09:47:24.050918+0000","last_became_peered":"2024-09-21T09:47:24.050918+0000","last_unstale":"2024-09-21T09:47:45.475132+0000","last_undegraded":"2024-09-21T09:47:45.475132+0000","last_fullsized":"2024-09-21T09:47:45.475132+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475023+0000","last_change":"2024-09-21T09:47:20.865532+0000","last_active":"2024-09-21T09:47:45.475023+0000","last_peered":"2024-09-21T09:47:45.475023+0000","last_clean":"2024-09-21T09:47:45.475023+0000","last_became_active":"2024-09-21T09:47:20.865103+0000","last_became_peered":"2024-09-21T09:47:20.865103+0000","last_unstale":"2024-09-21T09:47:45.475023+0000","last_undegraded":"2024-09-21T09:47:45.475023+0000","last_fullsized":"2024-09-21T09:47:45.475023+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474292+0000","last_change":"2024-09-21T09:47:24.067330+0000","last_active":"2024-09-21T09:47:45.474292+0000","last_peered":"2024-09-21T09:47:45.474292+0000","last_clean":"2024-09-21T09:47:45.474292+0000","last_became_active":"2024-09-21T09:47:24.066939+0000","last_became_peered":"2024-09-21T09:47:24.066939+0000","last_unstale":"2024-09-21T09:47:45.474292+0000","last_undegraded":"2024-09-21T09:47:45.474292+0000","last_fullsized":"2024-09-21T09:47:45.474292+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474434+0000","last_change":"2024-09-21T09:47:20.860605+0000","last_active":"2024-09-21T09:47:45.474434+0000","last_peered":"2024-09-21T09:47:45.474434+0000","last_clean":"2024-09-21T09:47:45.474434+0000","last_became_active":"2024-09-21T09:47:20.860288+0000","last_became_peered":"2024-09-21T09:47:20.860288+0000","last_unstale":"2024-09-21T09:47:45.474434+0000","last_undegraded":"2024-09-21T09:47:45.474434+0000","last_fullsized":"2024-09-21T09:47:45.474434+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473649+0000","last_change":"2024-09-21T09:47:24.057860+0000","last_active":"2024-09-21T09:47:45.473649+0000","last_peered":"2024-09-21T09:47:45.473649+0000","last_clean":"2024-09-21T09:47:45.473649+0000","last_became_active":"2024-09-21T09:47:24.057594+0000","last_became_peered":"2024-09-21T09:47:24.057594+0000","last_unstale":"2024-09-21T09:47:45.473649+0000","last_undegraded":"2024-09-21T09:47:45.473649+0000","last_fullsized":"2024-09-21T09:47:45.473649+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475431+0000","last_change":"2024-09-21T09:47:20.857026+0000","last_active":"2024-09-21T09:47:45.475431+0000","last_peered":"2024-09-21T09:47:45.475431+0000","last_clean":"2024-09-21T09:47:45.475431+0000","last_became_active":"2024-09-21T09:47:20.856719+0000","last_became_peered":"2024-09-21T09:47:20.856719+0000","last_unstale":"2024-09-21T09:47:45.475431+0000","last_undegraded":"2024-09-21T09:47:45.475431+0000","last_fullsized":"2024-09-21T09:47:45.475431+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471636+0000","last_change":"2024-09-21T09:47:24.059537+0000","last_active":"2024-09-21T09:47:45.471636+0000","last_peered":"2024-09-21T09:47:45.471636+0000","last_clean":"2024-09-21T09:47:45.471636+0000","last_became_active":"2024-09-21T09:47:24.059264+0000","last_became_peered":"2024-09-21T09:47:24.059264+0000","last_unstale":"2024-09-21T09:47:45.471636+0000","last_undegraded":"2024-09-21T09:47:45.471636+0000","last_fullsized":"2024-09-21T09:47:45.471636+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474711+0000","last_change":"2024-09-21T09:47:20.864446+0000","last_active":"2024-09-21T09:47:45.474711+0000","last_peered":"2024-09-21T09:47:45.474711+0000","last_clean":"2024-09-21T09:47:45.474711+0000","last_became_active":"2024-09-21T09:47:20.864160+0000","last_became_peered":"2024-09-21T09:47:20.864160+0000","last_unstale":"2024-09-21T09:47:45.474711+0000","last_undegraded":"2024-09-21T09:47:45.474711+0000","last_fullsized":"2024-09-21T09:47:45.474711+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475529+0000","last_change":"2024-09-21T09:47:24.070010+0000","last_active":"2024-09-21T09:47:45.475529+0000","last_peered":"2024-09-21T09:47:45.475529+0000","last_clean":"2024-09-21T09:47:45.475529+0000","last_became_active":"2024-09-21T09:47:24.069698+0000","last_became_peered":"2024-09-21T09:47:24.069698+0000","last_unstale":"2024-09-21T09:47:45.475529+0000","last_undegraded":"2024-09-21T09:47:45.475529+0000","last_fullsized":"2024-09-21T09:47:45.475529+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473662+0000","last_change":"2024-09-21T09:47:20.858242+0000","last_active":"2024-09-21T09:47:45.473662+0000","last_peered":"2024-09-21T09:47:45.473662+0000","last_clean":"2024-09-21T09:47:45.473662+0000","last_became_active":"2024-09-21T09:47:20.857951+0000","last_became_peered":"2024-09-21T09:47:20.857951+0000","last_unstale":"2024-09-21T09:47:45.473662+0000","last_undegraded":"2024-09-21T09:47:45.473662+0000","last_fullsized":"2024-09-21T09:47:45.473662+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473418+0000","last_change":"2024-09-21T09:47:24.063666+0000","last_active":"2024-09-21T09:47:45.473418+0000","last_peered":"2024-09-21T09:47:45.473418+0000","last_clean":"2024-09-21T09:47:45.473418+0000","last_became_active":"2024-09-21T09:47:24.063116+0000","last_became_peered":"2024-09-21T09:47:24.063116+0000","last_unstale":"2024-09-21T09:47:45.473418+0000","last_undegraded":"2024-09-21T09:47:45.473418+0000","last_fullsized":"2024-09-21T09:47:45.473418+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475534+0000","last_change":"2024-09-21T09:47:20.867134+0000","last_active":"2024-09-21T09:47:45.475534+0000","last_peered":"2024-09-21T09:47:45.475534+0000","last_clean":"2024-09-21T09:47:45.475534+0000","last_became_active":"2024-09-21T09:47:20.866875+0000","last_became_peered":"2024-09-21T09:47:20.866875+0000","last_unstale":"2024-09-21T09:47:45.475534+0000","last_undegraded":"2024-09-21T09:47:45.475534+0000","last_fullsized":"2024-09-21T09:47:45.475534+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475144+0000","last_change":"2024-09-21T09:47:24.069831+0000","last_active":"2024-09-21T09:47:45.475144+0000","last_peered":"2024-09-21T09:47:45.475144+0000","last_clean":"2024-09-21T09:47:45.475144+0000","last_became_active":"2024-09-21T09:47:24.069569+0000","last_became_peered":"2024-09-21T09:47:24.069569+0000","last_unstale":"2024-09-21T09:47:45.475144+0000","last_undegraded":"2024-09-21T09:47:45.475144+0000","last_fullsized":"2024-09-21T09:47:45.475144+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474104+0000","last_change":"2024-09-21T09:47:20.862992+0000","last_active":"2024-09-21T09:47:45.474104+0000","last_peered":"2024-09-21T09:47:45.474104+0000","last_clean":"2024-09-21T09:47:45.474104+0000","last_became_active":"2024-09-21T09:47:20.862582+0000","last_became_peered":"2024-09-21T09:47:20.862582+0000","last_unstale":"2024-09-21T09:47:45.474104+0000","last_undegraded":"2024-09-21T09:47:45.474104+0000","last_fullsized":"2024-09-21T09:47:45.474104+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473661+0000","last_change":"2024-09-21T09:47:24.058311+0000","last_active":"2024-09-21T09:47:45.473661+0000","last_peered":"2024-09-21T09:47:45.473661+0000","last_clean":"2024-09-21T09:47:45.473661+0000","last_became_active":"2024-09-21T09:47:24.057961+0000","last_became_peered":"2024-09-21T09:47:24.057961+0000","last_unstale":"2024-09-21T09:47:45.473661+0000","last_undegraded":"2024-09-21T09:47:45.473661+0000","last_fullsized":"2024-09-21T09:47:45.473661+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469194+0000","last_change":"2024-09-21T09:47:20.856509+0000","last_active":"2024-09-21T09:47:45.469194+0000","last_peered":"2024-09-21T09:47:45.469194+0000","last_clean":"2024-09-21T09:47:45.469194+0000","last_became_active":"2024-09-21T09:47:20.856229+0000","last_became_peered":"2024-09-21T09:47:20.856229+0000","last_unstale":"2024-09-21T09:47:45.469194+0000","last_undegraded":"2024-09-21T09:47:45.469194+0000","last_fullsized":"2024-09-21T09:47:45.469194+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474200+0000","last_change":"2024-09-21T09:47:24.069301+0000","last_active":"2024-09-21T09:47:45.474200+0000","last_peered":"2024-09-21T09:47:45.474200+0000","last_clean":"2024-09-21T09:47:45.474200+0000","last_became_active":"2024-09-21T09:47:24.068917+0000","last_became_peered":"2024-09-21T09:47:24.068917+0000","last_unstale":"2024-09-21T09:47:45.474200+0000","last_undegraded":"2024-09-21T09:47:45.474200+0000","last_fullsized":"2024-09-21T09:47:45.474200+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471246+0000","last_change":"2024-09-21T09:47:20.862118+0000","last_active":"2024-09-21T09:47:45.471246+0000","last_peered":"2024-09-21T09:47:45.471246+0000","last_clean":"2024-09-21T09:47:45.471246+0000","last_became_active":"2024-09-21T09:47:20.861815+0000","last_became_peered":"2024-09-21T09:47:20.861815+0000","last_unstale":"2024-09-21T09:47:45.471246+0000","last_undegraded":"2024-09-21T09:47:45.471246+0000","last_fullsized":"2024-09-21T09:47:45.471246+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474780+0000","last_change":"2024-09-21T09:47:24.069502+0000","last_active":"2024-09-21T09:47:45.474780+0000","last_peered":"2024-09-21T09:47:45.474780+0000","last_clean":"2024-09-21T09:47:45.474780+0000","last_became_active":"2024-09-21T09:47:24.069235+0000","last_became_peered":"2024-09-21T09:47:24.069235+0000","last_unstale":"2024-09-21T09:47:45.474780+0000","last_undegraded":"2024-09-21T09:47:45.474780+0000","last_fullsized":"2024-09-21T09:47:45.474780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469921+0000","last_change":"2024-09-21T09:47:20.862503+0000","last_active":"2024-09-21T09:47:45.469921+0000","last_peered":"2024-09-21T09:47:45.469921+0000","last_clean":"2024-09-21T09:47:45.469921+0000","last_became_active":"2024-09-21T09:47:20.862237+0000","last_became_peered":"2024-09-21T09:47:20.862237+0000","last_unstale":"2024-09-21T09:47:45.469921+0000","last_undegraded":"2024-09-21T09:47:45.469921+0000","last_fullsized":"2024-09-21T09:47:45.469921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476257+0000","last_change":"2024-09-21T09:47:24.069273+0000","last_active":"2024-09-21T09:47:45.476257+0000","last_peered":"2024-09-21T09:47:45.476257+0000","last_clean":"2024-09-21T09:47:45.476257+0000","last_became_active":"2024-09-21T09:47:24.068990+0000","last_became_peered":"2024-09-21T09:47:24.068990+0000","last_unstale":"2024-09-21T09:47:45.476257+0000","last_undegraded":"2024-09-21T09:47:45.476257+0000","last_fullsized":"2024-09-21T09:47:45.476257+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475558+0000","last_change":"2024-09-21T09:47:20.858095+0000","last_active":"2024-09-21T09:47:45.475558+0000","last_peered":"2024-09-21T09:47:45.475558+0000","last_clean":"2024-09-21T09:47:45.475558+0000","last_became_active":"2024-09-21T09:47:20.857760+0000","last_became_peered":"2024-09-21T09:47:20.857760+0000","last_unstale":"2024-09-21T09:47:45.475558+0000","last_undegraded":"2024-09-21T09:47:45.475558+0000","last_fullsized":"2024-09-21T09:47:45.475558+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473915+0000","last_change":"2024-09-21T09:47:24.063342+0000","last_active":"2024-09-21T09:47:45.473915+0000","last_peered":"2024-09-21T09:47:45.473915+0000","last_clean":"2024-09-21T09:47:45.473915+0000","last_became_active":"2024-09-21T09:47:24.063070+0000","last_became_peered":"2024-09-21T09:47:24.063070+0000","last_unstale":"2024-09-21T09:47:45.473915+0000","last_undegraded":"2024-09-21T09:47:45.473915+0000","last_fullsized":"2024-09-21T09:47:45.473915+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469526+0000","last_change":"2024-09-21T09:47:20.859702+0000","last_active":"2024-09-21T09:47:45.469526+0000","last_peered":"2024-09-21T09:47:45.469526+0000","last_clean":"2024-09-21T09:47:45.469526+0000","last_became_active":"2024-09-21T09:47:20.859443+0000","last_became_peered":"2024-09-21T09:47:20.859443+0000","last_unstale":"2024-09-21T09:47:45.469526+0000","last_undegraded":"2024-09-21T09:47:45.469526+0000","last_fullsized":"2024-09-21T09:47:45.469526+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475617+0000","last_change":"2024-09-21T09:47:24.070338+0000","last_active":"2024-09-21T09:47:45.475617+0000","last_peered":"2024-09-21T09:47:45.475617+0000","last_clean":"2024-09-21T09:47:45.475617+0000","last_became_active":"2024-09-21T09:47:24.070074+0000","last_became_peered":"2024-09-21T09:47:24.070074+0000","last_unstale":"2024-09-21T09:47:45.475617+0000","last_undegraded":"2024-09-21T09:47:45.475617+0000","last_fullsized":"2024-09-21T09:47:45.475617+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":25,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473341+0000","last_change":"2024-09-21T09:46:57.426595+0000","last_active":"2024-09-21T09:47:45.473341+0000","last_peered":"2024-09-21T09:47:45.473341+0000","last_clean":"2024-09-21T09:47:45.473341+0000","last_became_active":"2024-09-21T09:46:57.418997+0000","last_became_peered":"2024-09-21T09:46:57.418997+0000","last_unstale":"2024-09-21T09:47:45.473341+0000","last_undegraded":"2024-09-21T09:47:45.473341+0000","last_fullsized":"2024-09-21T09:47:45.473341+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_clean_scrub_stamp":"2024-09-21T09:44:57.819716+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475502+0000","last_change":"2024-09-21T09:47:20.866500+0000","last_active":"2024-09-21T09:47:45.475502+0000","last_peered":"2024-09-21T09:47:45.475502+0000","last_clean":"2024-09-21T09:47:45.475502+0000","last_became_active":"2024-09-21T09:47:20.866224+0000","last_became_peered":"2024-09-21T09:47:20.866224+0000","last_unstale":"2024-09-21T09:47:45.475502+0000","last_undegraded":"2024-09-21T09:47:45.475502+0000","last_fullsized":"2024-09-21T09:47:45.475502+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469992+0000","last_change":"2024-09-21T09:47:24.061053+0000","last_active":"2024-09-21T09:47:45.469992+0000","last_peered":"2024-09-21T09:47:45.469992+0000","last_clean":"2024-09-21T09:47:45.469992+0000","last_became_active":"2024-09-21T09:47:24.060632+0000","last_became_peered":"2024-09-21T09:47:24.060632+0000","last_unstale":"2024-09-21T09:47:45.469992+0000","last_undegraded":"2024-09-21T09:47:45.469992+0000","last_fullsized":"2024-09-21T09:47:45.469992+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473227+0000","last_change":"2024-09-21T09:47:20.869192+0000","last_active":"2024-09-21T09:47:45.473227+0000","last_peered":"2024-09-21T09:47:45.473227+0000","last_clean":"2024-09-21T09:47:45.473227+0000","last_became_active":"2024-09-21T09:47:20.868800+0000","last_became_peered":"2024-09-21T09:47:20.868800+0000","last_unstale":"2024-09-21T09:47:45.473227+0000","last_undegraded":"2024-09-21T09:47:45.473227+0000","last_fullsized":"2024-09-21T09:47:45.473227+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473014+0000","last_change":"2024-09-21T09:47:24.056658+0000","last_active":"2024-09-21T09:47:45.473014+0000","last_peered":"2024-09-21T09:47:45.473014+0000","last_clean":"2024-09-21T09:47:45.473014+0000","last_became_active":"2024-09-21T09:47:24.056366+0000","last_became_peered":"2024-09-21T09:47:24.056366+0000","last_unstale":"2024-09-21T09:47:45.473014+0000","last_undegraded":"2024-09-21T09:47:45.473014+0000","last_fullsized":"2024-09-21T09:47:45.473014+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469527+0000","last_change":"2024-09-21T09:47:20.861899+0000","last_active":"2024-09-21T09:47:45.469527+0000","last_peered":"2024-09-21T09:47:45.469527+0000","last_clean":"2024-09-21T09:47:45.469527+0000","last_became_active":"2024-09-21T09:47:20.861684+0000","last_became_peered":"2024-09-21T09:47:20.861684+0000","last_unstale":"2024-09-21T09:47:45.469527+0000","last_undegraded":"2024-09-21T09:47:45.469527+0000","last_fullsized":"2024-09-21T09:47:45.469527+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475106+0000","last_change":"2024-09-21T09:47:24.058235+0000","last_active":"2024-09-21T09:47:45.475106+0000","last_peered":"2024-09-21T09:47:45.475106+0000","last_clean":"2024-09-21T09:47:45.475106+0000","last_became_active":"2024-09-21T09:47:24.057972+0000","last_became_peered":"2024-09-21T09:47:24.057972+0000","last_unstale":"2024-09-21T09:47:45.475106+0000","last_undegraded":"2024-09-21T09:47:45.475106+0000","last_fullsized":"2024-09-21T09:47:45.475106+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475569+0000","last_change":"2024-09-21T09:47:20.861547+0000","last_active":"2024-09-21T09:47:45.475569+0000","last_peered":"2024-09-21T09:47:45.475569+0000","last_clean":"2024-09-21T09:47:45.475569+0000","last_became_active":"2024-09-21T09:47:20.861254+0000","last_became_peered":"2024-09-21T09:47:20.861254+0000","last_unstale":"2024-09-21T09:47:45.475569+0000","last_undegraded":"2024-09-21T09:47:45.475569+0000","last_fullsized":"2024-09-21T09:47:45.475569+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471048+0000","last_change":"2024-09-21T09:47:24.060191+0000","last_active":"2024-09-21T09:47:45.471048+0000","last_peered":"2024-09-21T09:47:45.471048+0000","last_clean":"2024-09-21T09:47:45.471048+0000","last_became_active":"2024-09-21T09:47:24.059929+0000","last_became_peered":"2024-09-21T09:47:24.059929+0000","last_unstale":"2024-09-21T09:47:45.471048+0000","last_undegraded":"2024-09-21T09:47:45.471048+0000","last_fullsized":"2024-09-21T09:47:45.471048+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475637+0000","last_change":"2024-09-21T09:47:20.862181+0000","last_active":"2024-09-21T09:47:45.475637+0000","last_peered":"2024-09-21T09:47:45.475637+0000","last_clean":"2024-09-21T09:47:45.475637+0000","last_became_active":"2024-09-21T09:47:20.861903+0000","last_became_peered":"2024-09-21T09:47:20.861903+0000","last_unstale":"2024-09-21T09:47:45.475637+0000","last_undegraded":"2024-09-21T09:47:45.475637+0000","last_fullsized":"2024-09-21T09:47:45.475637+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474023+0000","last_change":"2024-09-21T09:47:24.058849+0000","last_active":"2024-09-21T09:47:45.474023+0000","last_peered":"2024-09-21T09:47:45.474023+0000","last_clean":"2024-09-21T09:47:45.474023+0000","last_became_active":"2024-09-21T09:47:24.058564+0000","last_became_peered":"2024-09-21T09:47:24.058564+0000","last_unstale":"2024-09-21T09:47:45.474023+0000","last_undegraded":"2024-09-21T09:47:45.474023+0000","last_fullsized":"2024-09-21T09:47:45.474023+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476182+0000","last_change":"2024-09-21T09:47:20.866885+0000","last_active":"2024-09-21T09:47:45.476182+0000","last_peered":"2024-09-21T09:47:45.476182+0000","last_clean":"2024-09-21T09:47:45.476182+0000","last_became_active":"2024-09-21T09:47:20.866436+0000","last_became_peered":"2024-09-21T09:47:20.866436+0000","last_unstale":"2024-09-21T09:47:45.476182+0000","last_undegraded":"2024-09-21T09:47:45.476182+0000","last_fullsized":"2024-09-21T09:47:45.476182+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475549+0000","last_change":"2024-09-21T09:47:24.060367+0000","last_active":"2024-09-21T09:47:45.475549+0000","last_peered":"2024-09-21T09:47:45.475549+0000","last_clean":"2024-09-21T09:47:45.475549+0000","last_became_active":"2024-09-21T09:47:24.060083+0000","last_became_peered":"2024-09-21T09:47:24.060083+0000","last_unstale":"2024-09-21T09:47:45.475549+0000","last_undegraded":"2024-09-21T09:47:45.475549+0000","last_fullsized":"2024-09-21T09:47:45.475549+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475765+0000","last_change":"2024-09-21T09:47:20.858947+0000","last_active":"2024-09-21T09:47:45.475765+0000","last_peered":"2024-09-21T09:47:45.475765+0000","last_clean":"2024-09-21T09:47:45.475765+0000","last_became_active":"2024-09-21T09:47:20.858680+0000","last_became_peered":"2024-09-21T09:47:20.858680+0000","last_unstale":"2024-09-21T09:47:45.475765+0000","last_undegraded":"2024-09-21T09:47:45.475765+0000","last_fullsized":"2024-09-21T09:47:45.475765+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470043+0000","last_change":"2024-09-21T09:47:24.060498+0000","last_active":"2024-09-21T09:47:45.470043+0000","last_peered":"2024-09-21T09:47:45.470043+0000","last_clean":"2024-09-21T09:47:45.470043+0000","last_became_active":"2024-09-21T09:47:24.060179+0000","last_became_peered":"2024-09-21T09:47:24.060179+0000","last_unstale":"2024-09-21T09:47:45.470043+0000","last_undegraded":"2024-09-21T09:47:45.470043+0000","last_fullsized":"2024-09-21T09:47:45.470043+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475113+0000","last_change":"2024-09-21T09:47:20.865797+0000","last_active":"2024-09-21T09:47:45.475113+0000","last_peered":"2024-09-21T09:47:45.475113+0000","last_clean":"2024-09-21T09:47:45.475113+0000","last_became_active":"2024-09-21T09:47:20.865466+0000","last_became_peered":"2024-09-21T09:47:20.865466+0000","last_unstale":"2024-09-21T09:47:45.475113+0000","last_undegraded":"2024-09-21T09:47:45.475113+0000","last_fullsized":"2024-09-21T09:47:45.475113+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473171+0000","last_change":"2024-09-21T09:47:24.058690+0000","last_active":"2024-09-21T09:47:45.473171+0000","last_peered":"2024-09-21T09:47:45.473171+0000","last_clean":"2024-09-21T09:47:45.473171+0000","last_became_active":"2024-09-21T09:47:24.058371+0000","last_became_peered":"2024-09-21T09:47:24.058371+0000","last_unstale":"2024-09-21T09:47:45.473171+0000","last_undegraded":"2024-09-21T09:47:45.473171+0000","last_fullsized":"2024-09-21T09:47:45.473171+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473115+0000","last_change":"2024-09-21T09:47:20.868994+0000","last_active":"2024-09-21T09:47:45.473115+0000","last_peered":"2024-09-21T09:47:45.473115+0000","last_clean":"2024-09-21T09:47:45.473115+0000","last_became_active":"2024-09-21T09:47:20.868686+0000","last_became_peered":"2024-09-21T09:47:20.868686+0000","last_unstale":"2024-09-21T09:47:45.473115+0000","last_undegraded":"2024-09-21T09:47:45.473115+0000","last_fullsized":"2024-09-21T09:47:45.473115+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470397+0000","last_change":"2024-09-21T09:47:24.055068+0000","last_active":"2024-09-21T09:47:45.470397+0000","last_peered":"2024-09-21T09:47:45.470397+0000","last_clean":"2024-09-21T09:47:45.470397+0000","last_became_active":"2024-09-21T09:47:24.054785+0000","last_became_peered":"2024-09-21T09:47:24.054785+0000","last_unstale":"2024-09-21T09:47:45.470397+0000","last_undegraded":"2024-09-21T09:47:45.470397+0000","last_fullsized":"2024-09-21T09:47:45.470397+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475383+0000","last_change":"2024-09-21T09:47:20.862674+0000","last_active":"2024-09-21T09:47:45.475383+0000","last_peered":"2024-09-21T09:47:45.475383+0000","last_clean":"2024-09-21T09:47:45.475383+0000","last_became_active":"2024-09-21T09:47:20.861988+0000","last_became_peered":"2024-09-21T09:47:20.861988+0000","last_unstale":"2024-09-21T09:47:45.475383+0000","last_undegraded":"2024-09-21T09:47:45.475383+0000","last_fullsized":"2024-09-21T09:47:45.475383+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469755+0000","last_change":"2024-09-21T09:47:24.058826+0000","last_active":"2024-09-21T09:47:45.469755+0000","last_peered":"2024-09-21T09:47:45.469755+0000","last_clean":"2024-09-21T09:47:45.469755+0000","last_became_active":"2024-09-21T09:47:24.058565+0000","last_became_peered":"2024-09-21T09:47:24.058565+0000","last_unstale":"2024-09-21T09:47:45.469755+0000","last_undegraded":"2024-09-21T09:47:45.469755+0000","last_fullsized":"2024-09-21T09:47:45.469755+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475408+0000","last_change":"2024-09-21T09:47:20.861142+0000","last_active":"2024-09-21T09:47:45.475408+0000","last_peered":"2024-09-21T09:47:45.475408+0000","last_clean":"2024-09-21T09:47:45.475408+0000","last_became_active":"2024-09-21T09:47:20.860916+0000","last_became_peered":"2024-09-21T09:47:20.860916+0000","last_unstale":"2024-09-21T09:47:45.475408+0000","last_undegraded":"2024-09-21T09:47:45.475408+0000","last_fullsized":"2024-09-21T09:47:45.475408+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.481406+0000","last_change":"2024-09-21T09:47:46.481406+0000","last_active":"2024-09-21T09:47:46.481406+0000","last_peered":"2024-09-21T09:47:46.481406+0000","last_clean":"2024-09-21T09:47:46.481406+0000","last_became_active":"2024-09-21T09:47:46.481006+0000","last_became_peered":"2024-09-21T09:47:46.481006+0000","last_unstale":"2024-09-21T09:47:46.481406+0000","last_undegraded":"2024-09-21T09:47:46.481406+0000","last_fullsized":"2024-09-21T09:47:46.481406+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:18:56.780478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475414+0000","last_change":"2024-09-21T09:47:20.865234+0000","last_active":"2024-09-21T09:47:45.475414+0000","last_peered":"2024-09-21T09:47:45.475414+0000","last_clean":"2024-09-21T09:47:45.475414+0000","last_became_active":"2024-09-21T09:47:20.863790+0000","last_became_peered":"2024-09-21T09:47:20.863790+0000","last_unstale":"2024-09-21T09:47:45.475414+0000","last_undegraded":"2024-09-21T09:47:45.475414+0000","last_fullsized":"2024-09-21T09:47:45.475414+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474777+0000","last_change":"2024-09-21T09:47:24.056216+0000","last_active":"2024-09-21T09:47:45.474777+0000","last_peered":"2024-09-21T09:47:45.474777+0000","last_clean":"2024-09-21T09:47:45.474777+0000","last_became_active":"2024-09-21T09:47:24.055928+0000","last_became_peered":"2024-09-21T09:47:24.055928+0000","last_unstale":"2024-09-21T09:47:45.474777+0000","last_undegraded":"2024-09-21T09:47:45.474777+0000","last_fullsized":"2024-09-21T09:47:45.474777+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473935+0000","last_change":"2024-09-21T09:47:24.063533+0000","last_active":"2024-09-21T09:47:45.473935+0000","last_peered":"2024-09-21T09:47:45.473935+0000","last_clean":"2024-09-21T09:47:45.473935+0000","last_became_active":"2024-09-21T09:47:24.063293+0000","last_became_peered":"2024-09-21T09:47:24.063293+0000","last_unstale":"2024-09-21T09:47:45.473935+0000","last_undegraded":"2024-09-21T09:47:45.473935+0000","last_fullsized":"2024-09-21T09:47:45.473935+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474478+0000","last_change":"2024-09-21T09:47:20.864537+0000","last_active":"2024-09-21T09:47:45.474478+0000","last_peered":"2024-09-21T09:47:45.474478+0000","last_clean":"2024-09-21T09:47:45.474478+0000","last_became_active":"2024-09-21T09:47:20.864212+0000","last_became_peered":"2024-09-21T09:47:20.864212+0000","last_unstale":"2024-09-21T09:47:45.474478+0000","last_undegraded":"2024-09-21T09:47:45.474478+0000","last_fullsized":"2024-09-21T09:47:45.474478+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774197+0000","last_change":"2024-09-21T09:47:46.774197+0000","last_active":"2024-09-21T09:47:46.774197+0000","last_peered":"2024-09-21T09:47:46.774197+0000","last_clean":"2024-09-21T09:47:46.774197+0000","last_became_active":"2024-09-21T09:47:46.773846+0000","last_became_peered":"2024-09-21T09:47:46.773846+0000","last_unstale":"2024-09-21T09:47:46.774197+0000","last_undegraded":"2024-09-21T09:47:46.774197+0000","last_fullsized":"2024-09-21T09:47:46.774197+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475812+0000","last_change":"2024-09-21T09:47:20.866709+0000","last_active":"2024-09-21T09:47:45.475812+0000","last_peered":"2024-09-21T09:47:45.475812+0000","last_clean":"2024-09-21T09:47:45.475812+0000","last_became_active":"2024-09-21T09:47:20.866026+0000","last_became_peered":"2024-09-21T09:47:20.866026+0000","last_unstale":"2024-09-21T09:47:45.475812+0000","last_undegraded":"2024-09-21T09:47:45.475812+0000","last_fullsized":"2024-09-21T09:47:45.475812+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473620+0000","last_change":"2024-09-21T09:47:24.060801+0000","last_active":"2024-09-21T09:47:45.473620+0000","last_peered":"2024-09-21T09:47:45.473620+0000","last_clean":"2024-09-21T09:47:45.473620+0000","last_became_active":"2024-09-21T09:47:24.060536+0000","last_became_peered":"2024-09-21T09:47:24.060536+0000","last_unstale":"2024-09-21T09:47:45.473620+0000","last_undegraded":"2024-09-21T09:47:45.473620+0000","last_fullsized":"2024-09-21T09:47:45.473620+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470652+0000","last_change":"2024-09-21T09:47:20.864529+0000","last_active":"2024-09-21T09:47:45.470652+0000","last_peered":"2024-09-21T09:47:45.470652+0000","last_clean":"2024-09-21T09:47:45.470652+0000","last_became_active":"2024-09-21T09:47:20.864185+0000","last_became_peered":"2024-09-21T09:47:20.864185+0000","last_unstale":"2024-09-21T09:47:45.470652+0000","last_undegraded":"2024-09-21T09:47:45.470652+0000","last_fullsized":"2024-09-21T09:47:45.470652+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470571+0000","last_change":"2024-09-21T09:47:24.065677+0000","last_active":"2024-09-21T09:47:45.470571+0000","last_peered":"2024-09-21T09:47:45.470571+0000","last_clean":"2024-09-21T09:47:45.470571+0000","last_became_active":"2024-09-21T09:47:24.065373+0000","last_became_peered":"2024-09-21T09:47:24.065373+0000","last_unstale":"2024-09-21T09:47:45.470571+0000","last_undegraded":"2024-09-21T09:47:45.470571+0000","last_fullsized":"2024-09-21T09:47:45.470571+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473774+0000","last_change":"2024-09-21T09:47:20.865947+0000","last_active":"2024-09-21T09:47:45.473774+0000","last_peered":"2024-09-21T09:47:45.473774+0000","last_clean":"2024-09-21T09:47:45.473774+0000","last_became_active":"2024-09-21T09:47:20.865683+0000","last_became_peered":"2024-09-21T09:47:20.865683+0000","last_unstale":"2024-09-21T09:47:45.473774+0000","last_undegraded":"2024-09-21T09:47:45.473774+0000","last_fullsized":"2024-09-21T09:47:45.473774+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430222,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298524,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446628,"statfs":{"total":95995035648,"available":95689347072,"internally_reserved":0,"allocated":1191936,"data_stored":841969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41799999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.496}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59499999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58499999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66800000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68600000000000005}]}]},{"osd":6,"up_from":42,"seq":180388626450,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298512,"kb_used_data":1160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446640,"statfs":{"total":95995035648,"available":95689359360,"internally_reserved":0,"allocated":1187840,"data_stored":842370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.498},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.57199999999999995}]},{"osd":1,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.626},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.61299999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.32000000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.41599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]}]},{"osd":5,"up_from":36,"seq":154618822679,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":588,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":602112,"data_stored":252336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.47199999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.45800000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.433},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.60299999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.375}]},{"osd":6,"last update":"Sat Sep 21 09:47:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.53400000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849018908,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":606208,"data_stored":251705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.56299999999999994,"15min":0.57299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68999999999999995,"5min":0.748,"15min":0.748},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.432,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.58699999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.38,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.65600000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.495},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.435,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67700000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.442}]},{"osd":2,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.71599999999999997,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.51900000000000002,"15min":0.51600000000000001},"min":{"1min":0.42399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.624,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.51800000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.64200000000000002,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.44900000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.68300000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:47:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.63}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60399999999999998}]}]},{"osd":3,"up_from":25,"seq":107374182433,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297952,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447200,"statfs":{"total":95995035648,"available":95689932800,"internally_reserved":0,"allocated":606208,"data_stored":252350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.39800000000000002,"15min":0.40000000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.55500000000000005,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41299999999999998,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.59499999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.47699999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44500000000000001,"15min":0.45000000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.434},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42799999999999999,"15min":0.43099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.69299999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65600000000000003,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.45500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:47:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.58399999999999996,"15min":0.57999999999999996},"min":{"1min":0.41799999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86899999999999999,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.85299999999999998,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.52400000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Sep 21 09:47:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.54200000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56100000000000005}]}]},{"osd":2,"up_from":18,"seq":77309411365,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":589824,"data_stored":251009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.35999999999999999,"15min":0.35399999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.20599999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.27300000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.57799999999999996,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.28799999999999998}]},{"osd":1,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59999999999999998,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.254},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.442,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63800000000000001,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.30599999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:47:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.46800000000000003,"15min":0.47499999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.56200000000000006,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63900000000000001,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.22}]},{"osd":4,"last update":"Sat Sep 21 09:47:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73099999999999998,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.75900000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.434}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:47:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.501},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.44400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574890,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":598016,"data_stored":251169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.34799999999999998,"15min":0.32400000000000001},"min":{"1min":0.24399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.52500000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.44},{"interface":"front","average":{"1min":0.379,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.28199999999999997,"5min":0.214,"15min":0.214},"max":{"1min":0.47899999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.60999999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.375,"15min":0.375},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.49399999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.496},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.41099999999999998,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.63500000000000001,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.54400000000000004}]},{"osd":3,"last update":"Sat Sep 21 09:47:22 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57299999999999995,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.65000000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.58099999999999996}]},{"osd":4,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71199999999999997,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.67600000000000005,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.67300000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.70599999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.77100000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80600000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738415,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298528,"kb_used_data":1168,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446624,"statfs":{"total":95995035648,"available":95689342976,"internally_reserved":0,"allocated":1196032,"data_stored":841778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:47:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.33800000000000002,"15min":0.316},"min":{"1min":0.24199999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.374,"15min":0.34999999999999998},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.29699999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:47:01 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.27500000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.59099999999999997,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.39500000000000002,"15min":0.38500000000000001},"min":{"1min":0.33300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.27600000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:47:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.317},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.33100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.32000000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:47:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.56000000000000005,"15min":0.55200000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.57799999999999996,"15min":0.56999999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.61299999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:47:06 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.495}]},{"osd":6,"last update":"Sat Sep 21 09:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.67500000000000004}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:47:54.370 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph status --format=json 2024-09-21T09:47:54.631 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:54.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:54 smithi033 ceph-mon[27969]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:54.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:54 smithi033 ceph-mon[27969]: from='client.24413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:54 smithi033 ceph-mon[39045]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:54.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:54 smithi033 ceph-mon[39045]: from='client.24413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:54 smithi105 ceph-mon[31286]: pgmap v203: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:54 smithi105 ceph-mon[31286]: from='client.24413 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:47:56.492 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:56.492 INFO:teuthology.orchestra.run.smithi033.stdout:{"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":14,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":261,"monmap":{"epoch":3,"min_mon_release_name":"quincy","num_mons":3},"osdmap":{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1726912015,"num_in_osds":8,"osd_in_since":1726911998,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":97}],"num_pgs":97,"num_pools":3,"num_objects":24,"data_bytes":592706,"bytes_used":2442530816,"bytes_avail":765517754368,"bytes_total":767960285184},"fsmap":{"epoch":7,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"b","status":"up:active","gid":24358}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","restful"],"services":{"dashboard":"https://172.21.15.33:8443/"}},"servicemap":{"epoch":12,"modified":"2024-09-21T09:47:33.379547+0000","services":{"mds":{"daemons":{"summary":"","a":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}},"b":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}},"osd":{"daemons":{"summary":"","7":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{}} 2024-09-21T09:47:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:56 smithi033 ceph-mon[27969]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:56 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/4180407749' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-21T09:47:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:56 smithi033 ceph-mon[39045]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:56 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/4180407749' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-21T09:47:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:56 smithi105 ceph-mon[31286]: pgmap v204: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:56 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/4180407749' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2024-09-21T09:47:57.111 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2024-09-21T09:47:57.112 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:47:57.368 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:47:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:47:58 smithi105 ceph-mon[31286]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:58.834 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:47:58 smithi033 ceph-mon[39045]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:58.834 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:47:58 smithi033 ceph-mon[27969]: pgmap v205: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:47:59.369 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:47:59.370 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-21T09:48:00.129 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":205,"stamp":"2024-09-21T09:47:57.385625+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385284,"kb_used_data":6424,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575932,"statfs":{"total":767960285184,"available":765517754368,"internally_reserved":0,"allocated":6578176,"data_stored":3784686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.912793"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473431+0000","last_change":"2024-09-21T09:47:24.066961+0000","last_active":"2024-09-21T09:47:45.473431+0000","last_peered":"2024-09-21T09:47:45.473431+0000","last_clean":"2024-09-21T09:47:45.473431+0000","last_became_active":"2024-09-21T09:47:24.066672+0000","last_became_peered":"2024-09-21T09:47:24.066672+0000","last_unstale":"2024-09-21T09:47:45.473431+0000","last_undegraded":"2024-09-21T09:47:45.473431+0000","last_fullsized":"2024-09-21T09:47:45.473431+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475762+0000","last_change":"2024-09-21T09:47:24.059165+0000","last_active":"2024-09-21T09:47:45.475762+0000","last_peered":"2024-09-21T09:47:45.475762+0000","last_clean":"2024-09-21T09:47:45.475762+0000","last_became_active":"2024-09-21T09:47:24.058702+0000","last_became_peered":"2024-09-21T09:47:24.058702+0000","last_unstale":"2024-09-21T09:47:45.475762+0000","last_undegraded":"2024-09-21T09:47:45.475762+0000","last_fullsized":"2024-09-21T09:47:45.475762+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.472960+0000","last_change":"2024-09-21T09:47:24.060359+0000","last_active":"2024-09-21T09:47:45.472960+0000","last_peered":"2024-09-21T09:47:45.472960+0000","last_clean":"2024-09-21T09:47:45.472960+0000","last_became_active":"2024-09-21T09:47:24.060088+0000","last_became_peered":"2024-09-21T09:47:24.060088+0000","last_unstale":"2024-09-21T09:47:45.472960+0000","last_undegraded":"2024-09-21T09:47:45.472960+0000","last_fullsized":"2024-09-21T09:47:45.472960+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473799+0000","last_change":"2024-09-21T09:47:24.065047+0000","last_active":"2024-09-21T09:47:45.473799+0000","last_peered":"2024-09-21T09:47:45.473799+0000","last_clean":"2024-09-21T09:47:45.473799+0000","last_became_active":"2024-09-21T09:47:24.064743+0000","last_became_peered":"2024-09-21T09:47:24.064743+0000","last_unstale":"2024-09-21T09:47:45.473799+0000","last_undegraded":"2024-09-21T09:47:45.473799+0000","last_fullsized":"2024-09-21T09:47:45.473799+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475055+0000","last_change":"2024-09-21T09:47:24.069692+0000","last_active":"2024-09-21T09:47:45.475055+0000","last_peered":"2024-09-21T09:47:45.475055+0000","last_clean":"2024-09-21T09:47:45.475055+0000","last_became_active":"2024-09-21T09:47:24.069090+0000","last_became_peered":"2024-09-21T09:47:24.069090+0000","last_unstale":"2024-09-21T09:47:45.475055+0000","last_undegraded":"2024-09-21T09:47:45.475055+0000","last_fullsized":"2024-09-21T09:47:45.475055+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475319+0000","last_change":"2024-09-21T09:47:24.053540+0000","last_active":"2024-09-21T09:47:45.475319+0000","last_peered":"2024-09-21T09:47:45.475319+0000","last_clean":"2024-09-21T09:47:45.475319+0000","last_became_active":"2024-09-21T09:47:24.053163+0000","last_became_peered":"2024-09-21T09:47:24.053163+0000","last_unstale":"2024-09-21T09:47:45.475319+0000","last_undegraded":"2024-09-21T09:47:45.475319+0000","last_fullsized":"2024-09-21T09:47:45.475319+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774760+0000","last_change":"2024-09-21T09:47:46.774760+0000","last_active":"2024-09-21T09:47:46.774760+0000","last_peered":"2024-09-21T09:47:46.774760+0000","last_clean":"2024-09-21T09:47:46.774760+0000","last_became_active":"2024-09-21T09:47:46.774445+0000","last_became_peered":"2024-09-21T09:47:46.774445+0000","last_unstale":"2024-09-21T09:47:46.774760+0000","last_undegraded":"2024-09-21T09:47:46.774760+0000","last_fullsized":"2024-09-21T09:47:46.774760+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469675+0000","last_change":"2024-09-21T09:47:24.064091+0000","last_active":"2024-09-21T09:47:45.469675+0000","last_peered":"2024-09-21T09:47:45.469675+0000","last_clean":"2024-09-21T09:47:45.469675+0000","last_became_active":"2024-09-21T09:47:24.063698+0000","last_became_peered":"2024-09-21T09:47:24.063698+0000","last_unstale":"2024-09-21T09:47:45.469675+0000","last_undegraded":"2024-09-21T09:47:45.469675+0000","last_fullsized":"2024-09-21T09:47:45.469675+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476570+0000","last_change":"2024-09-21T09:47:24.071747+0000","last_active":"2024-09-21T09:47:45.476570+0000","last_peered":"2024-09-21T09:47:45.476570+0000","last_clean":"2024-09-21T09:47:45.476570+0000","last_became_active":"2024-09-21T09:47:24.071475+0000","last_became_peered":"2024-09-21T09:47:24.071475+0000","last_unstale":"2024-09-21T09:47:45.476570+0000","last_undegraded":"2024-09-21T09:47:45.476570+0000","last_fullsized":"2024-09-21T09:47:45.476570+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475068+0000","last_change":"2024-09-21T09:47:24.058216+0000","last_active":"2024-09-21T09:47:45.475068+0000","last_peered":"2024-09-21T09:47:45.475068+0000","last_clean":"2024-09-21T09:47:45.475068+0000","last_became_active":"2024-09-21T09:47:24.057940+0000","last_became_peered":"2024-09-21T09:47:24.057940+0000","last_unstale":"2024-09-21T09:47:45.475068+0000","last_undegraded":"2024-09-21T09:47:45.475068+0000","last_fullsized":"2024-09-21T09:47:45.475068+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474927+0000","last_change":"2024-09-21T09:47:24.053543+0000","last_active":"2024-09-21T09:47:45.474927+0000","last_peered":"2024-09-21T09:47:45.474927+0000","last_clean":"2024-09-21T09:47:45.474927+0000","last_became_active":"2024-09-21T09:47:24.053264+0000","last_became_peered":"2024-09-21T09:47:24.053264+0000","last_unstale":"2024-09-21T09:47:45.474927+0000","last_undegraded":"2024-09-21T09:47:45.474927+0000","last_fullsized":"2024-09-21T09:47:45.474927+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.850968+0000","last_change":"2024-09-21T09:47:46.850968+0000","last_active":"2024-09-21T09:47:46.850968+0000","last_peered":"2024-09-21T09:47:46.850968+0000","last_clean":"2024-09-21T09:47:46.850968+0000","last_became_active":"2024-09-21T09:47:46.850604+0000","last_became_peered":"2024-09-21T09:47:46.850604+0000","last_unstale":"2024-09-21T09:47:46.850968+0000","last_undegraded":"2024-09-21T09:47:46.850968+0000","last_fullsized":"2024-09-21T09:47:46.850968+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470142+0000","last_change":"2024-09-21T09:47:24.064334+0000","last_active":"2024-09-21T09:47:45.470142+0000","last_peered":"2024-09-21T09:47:45.470142+0000","last_clean":"2024-09-21T09:47:45.470142+0000","last_became_active":"2024-09-21T09:47:24.064063+0000","last_became_peered":"2024-09-21T09:47:24.064063+0000","last_unstale":"2024-09-21T09:47:45.470142+0000","last_undegraded":"2024-09-21T09:47:45.470142+0000","last_fullsized":"2024-09-21T09:47:45.470142+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473158+0000","last_change":"2024-09-21T09:47:24.058842+0000","last_active":"2024-09-21T09:47:45.473158+0000","last_peered":"2024-09-21T09:47:45.473158+0000","last_clean":"2024-09-21T09:47:45.473158+0000","last_became_active":"2024-09-21T09:47:24.058245+0000","last_became_peered":"2024-09-21T09:47:24.058245+0000","last_unstale":"2024-09-21T09:47:45.473158+0000","last_undegraded":"2024-09-21T09:47:45.473158+0000","last_fullsized":"2024-09-21T09:47:45.473158+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475221+0000","last_change":"2024-09-21T09:47:24.052028+0000","last_active":"2024-09-21T09:47:45.475221+0000","last_peered":"2024-09-21T09:47:45.475221+0000","last_clean":"2024-09-21T09:47:45.475221+0000","last_became_active":"2024-09-21T09:47:24.051766+0000","last_became_peered":"2024-09-21T09:47:24.051766+0000","last_unstale":"2024-09-21T09:47:45.475221+0000","last_undegraded":"2024-09-21T09:47:45.475221+0000","last_fullsized":"2024-09-21T09:47:45.475221+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473412+0000","last_change":"2024-09-21T09:47:24.060117+0000","last_active":"2024-09-21T09:47:45.473412+0000","last_peered":"2024-09-21T09:47:45.473412+0000","last_clean":"2024-09-21T09:47:45.473412+0000","last_became_active":"2024-09-21T09:47:24.059852+0000","last_became_peered":"2024-09-21T09:47:24.059852+0000","last_unstale":"2024-09-21T09:47:45.473412+0000","last_undegraded":"2024-09-21T09:47:45.473412+0000","last_fullsized":"2024-09-21T09:47:45.473412+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.480400+0000","last_change":"2024-09-21T09:47:46.480400+0000","last_active":"2024-09-21T09:47:46.480400+0000","last_peered":"2024-09-21T09:47:46.480400+0000","last_clean":"2024-09-21T09:47:46.480400+0000","last_became_active":"2024-09-21T09:47:46.480103+0000","last_became_peered":"2024-09-21T09:47:46.480103+0000","last_unstale":"2024-09-21T09:47:46.480400+0000","last_undegraded":"2024-09-21T09:47:46.480400+0000","last_fullsized":"2024-09-21T09:47:46.480400+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469780+0000","last_change":"2024-09-21T09:47:24.057147+0000","last_active":"2024-09-21T09:47:45.469780+0000","last_peered":"2024-09-21T09:47:45.469780+0000","last_clean":"2024-09-21T09:47:45.469780+0000","last_became_active":"2024-09-21T09:47:24.056798+0000","last_became_peered":"2024-09-21T09:47:24.056798+0000","last_unstale":"2024-09-21T09:47:45.469780+0000","last_undegraded":"2024-09-21T09:47:45.469780+0000","last_fullsized":"2024-09-21T09:47:45.469780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475925+0000","last_change":"2024-09-21T09:47:24.070640+0000","last_active":"2024-09-21T09:47:45.475925+0000","last_peered":"2024-09-21T09:47:45.475925+0000","last_clean":"2024-09-21T09:47:45.475925+0000","last_became_active":"2024-09-21T09:47:24.070359+0000","last_became_peered":"2024-09-21T09:47:24.070359+0000","last_unstale":"2024-09-21T09:47:45.475925+0000","last_undegraded":"2024-09-21T09:47:45.475925+0000","last_fullsized":"2024-09-21T09:47:45.475925+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474605+0000","last_change":"2024-09-21T09:47:24.056659+0000","last_active":"2024-09-21T09:47:45.474605+0000","last_peered":"2024-09-21T09:47:45.474605+0000","last_clean":"2024-09-21T09:47:45.474605+0000","last_became_active":"2024-09-21T09:47:24.056360+0000","last_became_peered":"2024-09-21T09:47:24.056360+0000","last_unstale":"2024-09-21T09:47:45.474605+0000","last_undegraded":"2024-09-21T09:47:45.474605+0000","last_fullsized":"2024-09-21T09:47:45.474605+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473450+0000","last_change":"2024-09-21T09:47:24.065154+0000","last_active":"2024-09-21T09:47:45.473450+0000","last_peered":"2024-09-21T09:47:45.473450+0000","last_clean":"2024-09-21T09:47:45.473450+0000","last_became_active":"2024-09-21T09:47:24.064231+0000","last_became_peered":"2024-09-21T09:47:24.064231+0000","last_unstale":"2024-09-21T09:47:45.473450+0000","last_undegraded":"2024-09-21T09:47:45.473450+0000","last_fullsized":"2024-09-21T09:47:45.473450+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475326+0000","last_change":"2024-09-21T09:47:24.062716+0000","last_active":"2024-09-21T09:47:45.475326+0000","last_peered":"2024-09-21T09:47:45.475326+0000","last_clean":"2024-09-21T09:47:45.475326+0000","last_became_active":"2024-09-21T09:47:24.062437+0000","last_became_peered":"2024-09-21T09:47:24.062437+0000","last_unstale":"2024-09-21T09:47:45.475326+0000","last_undegraded":"2024-09-21T09:47:45.475326+0000","last_fullsized":"2024-09-21T09:47:45.475326+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470046+0000","last_change":"2024-09-21T09:47:24.051387+0000","last_active":"2024-09-21T09:47:45.470046+0000","last_peered":"2024-09-21T09:47:45.470046+0000","last_clean":"2024-09-21T09:47:45.470046+0000","last_became_active":"2024-09-21T09:47:24.051090+0000","last_became_peered":"2024-09-21T09:47:24.051090+0000","last_unstale":"2024-09-21T09:47:45.470046+0000","last_undegraded":"2024-09-21T09:47:45.470046+0000","last_fullsized":"2024-09-21T09:47:45.470046+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473500+0000","last_change":"2024-09-21T09:47:24.058079+0000","last_active":"2024-09-21T09:47:45.473500+0000","last_peered":"2024-09-21T09:47:45.473500+0000","last_clean":"2024-09-21T09:47:45.473500+0000","last_became_active":"2024-09-21T09:47:24.057753+0000","last_became_peered":"2024-09-21T09:47:24.057753+0000","last_unstale":"2024-09-21T09:47:45.473500+0000","last_undegraded":"2024-09-21T09:47:45.473500+0000","last_fullsized":"2024-09-21T09:47:45.473500+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469933+0000","last_change":"2024-09-21T09:47:24.057220+0000","last_active":"2024-09-21T09:47:45.469933+0000","last_peered":"2024-09-21T09:47:45.469933+0000","last_clean":"2024-09-21T09:47:45.469933+0000","last_became_active":"2024-09-21T09:47:24.056915+0000","last_became_peered":"2024-09-21T09:47:24.056915+0000","last_unstale":"2024-09-21T09:47:45.469933+0000","last_undegraded":"2024-09-21T09:47:45.469933+0000","last_fullsized":"2024-09-21T09:47:45.469933+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475187+0000","last_change":"2024-09-21T09:47:24.057310+0000","last_active":"2024-09-21T09:47:45.475187+0000","last_peered":"2024-09-21T09:47:45.475187+0000","last_clean":"2024-09-21T09:47:45.475187+0000","last_became_active":"2024-09-21T09:47:24.057047+0000","last_became_peered":"2024-09-21T09:47:24.057047+0000","last_unstale":"2024-09-21T09:47:45.475187+0000","last_undegraded":"2024-09-21T09:47:45.475187+0000","last_fullsized":"2024-09-21T09:47:45.475187+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475460+0000","last_change":"2024-09-21T09:47:24.060976+0000","last_active":"2024-09-21T09:47:45.475460+0000","last_peered":"2024-09-21T09:47:45.475460+0000","last_clean":"2024-09-21T09:47:45.475460+0000","last_became_active":"2024-09-21T09:47:24.060725+0000","last_became_peered":"2024-09-21T09:47:24.060725+0000","last_unstale":"2024-09-21T09:47:45.475460+0000","last_undegraded":"2024-09-21T09:47:45.475460+0000","last_fullsized":"2024-09-21T09:47:45.475460+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470271+0000","last_change":"2024-09-21T09:47:24.053563+0000","last_active":"2024-09-21T09:47:45.470271+0000","last_peered":"2024-09-21T09:47:45.470271+0000","last_clean":"2024-09-21T09:47:45.470271+0000","last_became_active":"2024-09-21T09:47:24.052853+0000","last_became_peered":"2024-09-21T09:47:24.052853+0000","last_unstale":"2024-09-21T09:47:45.470271+0000","last_undegraded":"2024-09-21T09:47:45.470271+0000","last_fullsized":"2024-09-21T09:47:45.470271+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473532+0000","last_change":"2024-09-21T09:47:24.068204+0000","last_active":"2024-09-21T09:47:45.473532+0000","last_peered":"2024-09-21T09:47:45.473532+0000","last_clean":"2024-09-21T09:47:45.473532+0000","last_became_active":"2024-09-21T09:47:24.067911+0000","last_became_peered":"2024-09-21T09:47:24.067911+0000","last_unstale":"2024-09-21T09:47:45.473532+0000","last_undegraded":"2024-09-21T09:47:45.473532+0000","last_fullsized":"2024-09-21T09:47:45.473532+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475415+0000","last_change":"2024-09-21T09:47:24.058882+0000","last_active":"2024-09-21T09:47:45.475415+0000","last_peered":"2024-09-21T09:47:45.475415+0000","last_clean":"2024-09-21T09:47:45.475415+0000","last_became_active":"2024-09-21T09:47:24.058587+0000","last_became_peered":"2024-09-21T09:47:24.058587+0000","last_unstale":"2024-09-21T09:47:45.475415+0000","last_undegraded":"2024-09-21T09:47:45.475415+0000","last_fullsized":"2024-09-21T09:47:45.475415+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469392+0000","last_change":"2024-09-21T09:47:24.060446+0000","last_active":"2024-09-21T09:47:45.469392+0000","last_peered":"2024-09-21T09:47:45.469392+0000","last_clean":"2024-09-21T09:47:45.469392+0000","last_became_active":"2024-09-21T09:47:24.060165+0000","last_became_peered":"2024-09-21T09:47:24.060165+0000","last_unstale":"2024-09-21T09:47:45.469392+0000","last_undegraded":"2024-09-21T09:47:45.469392+0000","last_fullsized":"2024-09-21T09:47:45.469392+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473573+0000","last_change":"2024-09-21T09:47:24.059974+0000","last_active":"2024-09-21T09:47:45.473573+0000","last_peered":"2024-09-21T09:47:45.473573+0000","last_clean":"2024-09-21T09:47:45.473573+0000","last_became_active":"2024-09-21T09:47:24.059270+0000","last_became_peered":"2024-09-21T09:47:24.059270+0000","last_unstale":"2024-09-21T09:47:45.473573+0000","last_undegraded":"2024-09-21T09:47:45.473573+0000","last_fullsized":"2024-09-21T09:47:45.473573+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470251+0000","last_change":"2024-09-21T09:47:24.048941+0000","last_active":"2024-09-21T09:47:45.470251+0000","last_peered":"2024-09-21T09:47:45.470251+0000","last_clean":"2024-09-21T09:47:45.470251+0000","last_became_active":"2024-09-21T09:47:24.048631+0000","last_became_peered":"2024-09-21T09:47:24.048631+0000","last_unstale":"2024-09-21T09:47:45.470251+0000","last_undegraded":"2024-09-21T09:47:45.470251+0000","last_fullsized":"2024-09-21T09:47:45.470251+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470304+0000","last_change":"2024-09-21T09:47:20.866010+0000","last_active":"2024-09-21T09:47:45.470304+0000","last_peered":"2024-09-21T09:47:45.470304+0000","last_clean":"2024-09-21T09:47:45.470304+0000","last_became_active":"2024-09-21T09:47:20.865732+0000","last_became_peered":"2024-09-21T09:47:20.865732+0000","last_unstale":"2024-09-21T09:47:45.470304+0000","last_undegraded":"2024-09-21T09:47:45.470304+0000","last_fullsized":"2024-09-21T09:47:45.470304+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473645+0000","last_change":"2024-09-21T09:47:24.067813+0000","last_active":"2024-09-21T09:47:45.473645+0000","last_peered":"2024-09-21T09:47:45.473645+0000","last_clean":"2024-09-21T09:47:45.473645+0000","last_became_active":"2024-09-21T09:47:24.067412+0000","last_became_peered":"2024-09-21T09:47:24.067412+0000","last_unstale":"2024-09-21T09:47:45.473645+0000","last_undegraded":"2024-09-21T09:47:45.473645+0000","last_fullsized":"2024-09-21T09:47:45.473645+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":27,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.481320+0000","last_change":"2024-09-21T09:47:46.481320+0000","last_active":"2024-09-21T09:47:46.481320+0000","last_peered":"2024-09-21T09:47:46.481320+0000","last_clean":"2024-09-21T09:47:46.481320+0000","last_became_active":"2024-09-21T09:47:46.480905+0000","last_became_peered":"2024-09-21T09:47:46.480905+0000","last_unstale":"2024-09-21T09:47:46.481320+0000","last_undegraded":"2024-09-21T09:47:46.481320+0000","last_fullsized":"2024-09-21T09:47:46.481320+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:23.831301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470210+0000","last_change":"2024-09-21T09:47:20.859829+0000","last_active":"2024-09-21T09:47:45.470210+0000","last_peered":"2024-09-21T09:47:45.470210+0000","last_clean":"2024-09-21T09:47:45.470210+0000","last_became_active":"2024-09-21T09:47:20.859507+0000","last_became_peered":"2024-09-21T09:47:20.859507+0000","last_unstale":"2024-09-21T09:47:45.470210+0000","last_undegraded":"2024-09-21T09:47:45.470210+0000","last_fullsized":"2024-09-21T09:47:45.470210+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469715+0000","last_change":"2024-09-21T09:47:24.053702+0000","last_active":"2024-09-21T09:47:45.469715+0000","last_peered":"2024-09-21T09:47:45.469715+0000","last_clean":"2024-09-21T09:47:45.469715+0000","last_became_active":"2024-09-21T09:47:24.053295+0000","last_became_peered":"2024-09-21T09:47:24.053295+0000","last_unstale":"2024-09-21T09:47:45.469715+0000","last_undegraded":"2024-09-21T09:47:45.469715+0000","last_fullsized":"2024-09-21T09:47:45.469715+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475673+0000","last_change":"2024-09-21T09:47:20.863181+0000","last_active":"2024-09-21T09:47:45.475673+0000","last_peered":"2024-09-21T09:47:45.475673+0000","last_clean":"2024-09-21T09:47:45.475673+0000","last_became_active":"2024-09-21T09:47:20.862895+0000","last_became_peered":"2024-09-21T09:47:20.862895+0000","last_unstale":"2024-09-21T09:47:45.475673+0000","last_undegraded":"2024-09-21T09:47:45.475673+0000","last_fullsized":"2024-09-21T09:47:45.475673+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475132+0000","last_change":"2024-09-21T09:47:24.051248+0000","last_active":"2024-09-21T09:47:45.475132+0000","last_peered":"2024-09-21T09:47:45.475132+0000","last_clean":"2024-09-21T09:47:45.475132+0000","last_became_active":"2024-09-21T09:47:24.050918+0000","last_became_peered":"2024-09-21T09:47:24.050918+0000","last_unstale":"2024-09-21T09:47:45.475132+0000","last_undegraded":"2024-09-21T09:47:45.475132+0000","last_fullsized":"2024-09-21T09:47:45.475132+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475023+0000","last_change":"2024-09-21T09:47:20.865532+0000","last_active":"2024-09-21T09:47:45.475023+0000","last_peered":"2024-09-21T09:47:45.475023+0000","last_clean":"2024-09-21T09:47:45.475023+0000","last_became_active":"2024-09-21T09:47:20.865103+0000","last_became_peered":"2024-09-21T09:47:20.865103+0000","last_unstale":"2024-09-21T09:47:45.475023+0000","last_undegraded":"2024-09-21T09:47:45.475023+0000","last_fullsized":"2024-09-21T09:47:45.475023+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474292+0000","last_change":"2024-09-21T09:47:24.067330+0000","last_active":"2024-09-21T09:47:45.474292+0000","last_peered":"2024-09-21T09:47:45.474292+0000","last_clean":"2024-09-21T09:47:45.474292+0000","last_became_active":"2024-09-21T09:47:24.066939+0000","last_became_peered":"2024-09-21T09:47:24.066939+0000","last_unstale":"2024-09-21T09:47:45.474292+0000","last_undegraded":"2024-09-21T09:47:45.474292+0000","last_fullsized":"2024-09-21T09:47:45.474292+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474434+0000","last_change":"2024-09-21T09:47:20.860605+0000","last_active":"2024-09-21T09:47:45.474434+0000","last_peered":"2024-09-21T09:47:45.474434+0000","last_clean":"2024-09-21T09:47:45.474434+0000","last_became_active":"2024-09-21T09:47:20.860288+0000","last_became_peered":"2024-09-21T09:47:20.860288+0000","last_unstale":"2024-09-21T09:47:45.474434+0000","last_undegraded":"2024-09-21T09:47:45.474434+0000","last_fullsized":"2024-09-21T09:47:45.474434+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473649+0000","last_change":"2024-09-21T09:47:24.057860+0000","last_active":"2024-09-21T09:47:45.473649+0000","last_peered":"2024-09-21T09:47:45.473649+0000","last_clean":"2024-09-21T09:47:45.473649+0000","last_became_active":"2024-09-21T09:47:24.057594+0000","last_became_peered":"2024-09-21T09:47:24.057594+0000","last_unstale":"2024-09-21T09:47:45.473649+0000","last_undegraded":"2024-09-21T09:47:45.473649+0000","last_fullsized":"2024-09-21T09:47:45.473649+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475431+0000","last_change":"2024-09-21T09:47:20.857026+0000","last_active":"2024-09-21T09:47:45.475431+0000","last_peered":"2024-09-21T09:47:45.475431+0000","last_clean":"2024-09-21T09:47:45.475431+0000","last_became_active":"2024-09-21T09:47:20.856719+0000","last_became_peered":"2024-09-21T09:47:20.856719+0000","last_unstale":"2024-09-21T09:47:45.475431+0000","last_undegraded":"2024-09-21T09:47:45.475431+0000","last_fullsized":"2024-09-21T09:47:45.475431+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471636+0000","last_change":"2024-09-21T09:47:24.059537+0000","last_active":"2024-09-21T09:47:45.471636+0000","last_peered":"2024-09-21T09:47:45.471636+0000","last_clean":"2024-09-21T09:47:45.471636+0000","last_became_active":"2024-09-21T09:47:24.059264+0000","last_became_peered":"2024-09-21T09:47:24.059264+0000","last_unstale":"2024-09-21T09:47:45.471636+0000","last_undegraded":"2024-09-21T09:47:45.471636+0000","last_fullsized":"2024-09-21T09:47:45.471636+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474711+0000","last_change":"2024-09-21T09:47:20.864446+0000","last_active":"2024-09-21T09:47:45.474711+0000","last_peered":"2024-09-21T09:47:45.474711+0000","last_clean":"2024-09-21T09:47:45.474711+0000","last_became_active":"2024-09-21T09:47:20.864160+0000","last_became_peered":"2024-09-21T09:47:20.864160+0000","last_unstale":"2024-09-21T09:47:45.474711+0000","last_undegraded":"2024-09-21T09:47:45.474711+0000","last_fullsized":"2024-09-21T09:47:45.474711+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475529+0000","last_change":"2024-09-21T09:47:24.070010+0000","last_active":"2024-09-21T09:47:45.475529+0000","last_peered":"2024-09-21T09:47:45.475529+0000","last_clean":"2024-09-21T09:47:45.475529+0000","last_became_active":"2024-09-21T09:47:24.069698+0000","last_became_peered":"2024-09-21T09:47:24.069698+0000","last_unstale":"2024-09-21T09:47:45.475529+0000","last_undegraded":"2024-09-21T09:47:45.475529+0000","last_fullsized":"2024-09-21T09:47:45.475529+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473662+0000","last_change":"2024-09-21T09:47:20.858242+0000","last_active":"2024-09-21T09:47:45.473662+0000","last_peered":"2024-09-21T09:47:45.473662+0000","last_clean":"2024-09-21T09:47:45.473662+0000","last_became_active":"2024-09-21T09:47:20.857951+0000","last_became_peered":"2024-09-21T09:47:20.857951+0000","last_unstale":"2024-09-21T09:47:45.473662+0000","last_undegraded":"2024-09-21T09:47:45.473662+0000","last_fullsized":"2024-09-21T09:47:45.473662+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473418+0000","last_change":"2024-09-21T09:47:24.063666+0000","last_active":"2024-09-21T09:47:45.473418+0000","last_peered":"2024-09-21T09:47:45.473418+0000","last_clean":"2024-09-21T09:47:45.473418+0000","last_became_active":"2024-09-21T09:47:24.063116+0000","last_became_peered":"2024-09-21T09:47:24.063116+0000","last_unstale":"2024-09-21T09:47:45.473418+0000","last_undegraded":"2024-09-21T09:47:45.473418+0000","last_fullsized":"2024-09-21T09:47:45.473418+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475534+0000","last_change":"2024-09-21T09:47:20.867134+0000","last_active":"2024-09-21T09:47:45.475534+0000","last_peered":"2024-09-21T09:47:45.475534+0000","last_clean":"2024-09-21T09:47:45.475534+0000","last_became_active":"2024-09-21T09:47:20.866875+0000","last_became_peered":"2024-09-21T09:47:20.866875+0000","last_unstale":"2024-09-21T09:47:45.475534+0000","last_undegraded":"2024-09-21T09:47:45.475534+0000","last_fullsized":"2024-09-21T09:47:45.475534+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475144+0000","last_change":"2024-09-21T09:47:24.069831+0000","last_active":"2024-09-21T09:47:45.475144+0000","last_peered":"2024-09-21T09:47:45.475144+0000","last_clean":"2024-09-21T09:47:45.475144+0000","last_became_active":"2024-09-21T09:47:24.069569+0000","last_became_peered":"2024-09-21T09:47:24.069569+0000","last_unstale":"2024-09-21T09:47:45.475144+0000","last_undegraded":"2024-09-21T09:47:45.475144+0000","last_fullsized":"2024-09-21T09:47:45.475144+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474104+0000","last_change":"2024-09-21T09:47:20.862992+0000","last_active":"2024-09-21T09:47:45.474104+0000","last_peered":"2024-09-21T09:47:45.474104+0000","last_clean":"2024-09-21T09:47:45.474104+0000","last_became_active":"2024-09-21T09:47:20.862582+0000","last_became_peered":"2024-09-21T09:47:20.862582+0000","last_unstale":"2024-09-21T09:47:45.474104+0000","last_undegraded":"2024-09-21T09:47:45.474104+0000","last_fullsized":"2024-09-21T09:47:45.474104+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473661+0000","last_change":"2024-09-21T09:47:24.058311+0000","last_active":"2024-09-21T09:47:45.473661+0000","last_peered":"2024-09-21T09:47:45.473661+0000","last_clean":"2024-09-21T09:47:45.473661+0000","last_became_active":"2024-09-21T09:47:24.057961+0000","last_became_peered":"2024-09-21T09:47:24.057961+0000","last_unstale":"2024-09-21T09:47:45.473661+0000","last_undegraded":"2024-09-21T09:47:45.473661+0000","last_fullsized":"2024-09-21T09:47:45.473661+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469194+0000","last_change":"2024-09-21T09:47:20.856509+0000","last_active":"2024-09-21T09:47:45.469194+0000","last_peered":"2024-09-21T09:47:45.469194+0000","last_clean":"2024-09-21T09:47:45.469194+0000","last_became_active":"2024-09-21T09:47:20.856229+0000","last_became_peered":"2024-09-21T09:47:20.856229+0000","last_unstale":"2024-09-21T09:47:45.469194+0000","last_undegraded":"2024-09-21T09:47:45.469194+0000","last_fullsized":"2024-09-21T09:47:45.469194+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474200+0000","last_change":"2024-09-21T09:47:24.069301+0000","last_active":"2024-09-21T09:47:45.474200+0000","last_peered":"2024-09-21T09:47:45.474200+0000","last_clean":"2024-09-21T09:47:45.474200+0000","last_became_active":"2024-09-21T09:47:24.068917+0000","last_became_peered":"2024-09-21T09:47:24.068917+0000","last_unstale":"2024-09-21T09:47:45.474200+0000","last_undegraded":"2024-09-21T09:47:45.474200+0000","last_fullsized":"2024-09-21T09:47:45.474200+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471246+0000","last_change":"2024-09-21T09:47:20.862118+0000","last_active":"2024-09-21T09:47:45.471246+0000","last_peered":"2024-09-21T09:47:45.471246+0000","last_clean":"2024-09-21T09:47:45.471246+0000","last_became_active":"2024-09-21T09:47:20.861815+0000","last_became_peered":"2024-09-21T09:47:20.861815+0000","last_unstale":"2024-09-21T09:47:45.471246+0000","last_undegraded":"2024-09-21T09:47:45.471246+0000","last_fullsized":"2024-09-21T09:47:45.471246+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474780+0000","last_change":"2024-09-21T09:47:24.069502+0000","last_active":"2024-09-21T09:47:45.474780+0000","last_peered":"2024-09-21T09:47:45.474780+0000","last_clean":"2024-09-21T09:47:45.474780+0000","last_became_active":"2024-09-21T09:47:24.069235+0000","last_became_peered":"2024-09-21T09:47:24.069235+0000","last_unstale":"2024-09-21T09:47:45.474780+0000","last_undegraded":"2024-09-21T09:47:45.474780+0000","last_fullsized":"2024-09-21T09:47:45.474780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469921+0000","last_change":"2024-09-21T09:47:20.862503+0000","last_active":"2024-09-21T09:47:45.469921+0000","last_peered":"2024-09-21T09:47:45.469921+0000","last_clean":"2024-09-21T09:47:45.469921+0000","last_became_active":"2024-09-21T09:47:20.862237+0000","last_became_peered":"2024-09-21T09:47:20.862237+0000","last_unstale":"2024-09-21T09:47:45.469921+0000","last_undegraded":"2024-09-21T09:47:45.469921+0000","last_fullsized":"2024-09-21T09:47:45.469921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476257+0000","last_change":"2024-09-21T09:47:24.069273+0000","last_active":"2024-09-21T09:47:45.476257+0000","last_peered":"2024-09-21T09:47:45.476257+0000","last_clean":"2024-09-21T09:47:45.476257+0000","last_became_active":"2024-09-21T09:47:24.068990+0000","last_became_peered":"2024-09-21T09:47:24.068990+0000","last_unstale":"2024-09-21T09:47:45.476257+0000","last_undegraded":"2024-09-21T09:47:45.476257+0000","last_fullsized":"2024-09-21T09:47:45.476257+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475558+0000","last_change":"2024-09-21T09:47:20.858095+0000","last_active":"2024-09-21T09:47:45.475558+0000","last_peered":"2024-09-21T09:47:45.475558+0000","last_clean":"2024-09-21T09:47:45.475558+0000","last_became_active":"2024-09-21T09:47:20.857760+0000","last_became_peered":"2024-09-21T09:47:20.857760+0000","last_unstale":"2024-09-21T09:47:45.475558+0000","last_undegraded":"2024-09-21T09:47:45.475558+0000","last_fullsized":"2024-09-21T09:47:45.475558+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473915+0000","last_change":"2024-09-21T09:47:24.063342+0000","last_active":"2024-09-21T09:47:45.473915+0000","last_peered":"2024-09-21T09:47:45.473915+0000","last_clean":"2024-09-21T09:47:45.473915+0000","last_became_active":"2024-09-21T09:47:24.063070+0000","last_became_peered":"2024-09-21T09:47:24.063070+0000","last_unstale":"2024-09-21T09:47:45.473915+0000","last_undegraded":"2024-09-21T09:47:45.473915+0000","last_fullsized":"2024-09-21T09:47:45.473915+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469526+0000","last_change":"2024-09-21T09:47:20.859702+0000","last_active":"2024-09-21T09:47:45.469526+0000","last_peered":"2024-09-21T09:47:45.469526+0000","last_clean":"2024-09-21T09:47:45.469526+0000","last_became_active":"2024-09-21T09:47:20.859443+0000","last_became_peered":"2024-09-21T09:47:20.859443+0000","last_unstale":"2024-09-21T09:47:45.469526+0000","last_undegraded":"2024-09-21T09:47:45.469526+0000","last_fullsized":"2024-09-21T09:47:45.469526+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475617+0000","last_change":"2024-09-21T09:47:24.070338+0000","last_active":"2024-09-21T09:47:45.475617+0000","last_peered":"2024-09-21T09:47:45.475617+0000","last_clean":"2024-09-21T09:47:45.475617+0000","last_became_active":"2024-09-21T09:47:24.070074+0000","last_became_peered":"2024-09-21T09:47:24.070074+0000","last_unstale":"2024-09-21T09:47:45.475617+0000","last_undegraded":"2024-09-21T09:47:45.475617+0000","last_fullsized":"2024-09-21T09:47:45.475617+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":25,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473341+0000","last_change":"2024-09-21T09:46:57.426595+0000","last_active":"2024-09-21T09:47:45.473341+0000","last_peered":"2024-09-21T09:47:45.473341+0000","last_clean":"2024-09-21T09:47:45.473341+0000","last_became_active":"2024-09-21T09:46:57.418997+0000","last_became_peered":"2024-09-21T09:46:57.418997+0000","last_unstale":"2024-09-21T09:47:45.473341+0000","last_undegraded":"2024-09-21T09:47:45.473341+0000","last_fullsized":"2024-09-21T09:47:45.473341+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_clean_scrub_stamp":"2024-09-21T09:44:57.819716+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475502+0000","last_change":"2024-09-21T09:47:20.866500+0000","last_active":"2024-09-21T09:47:45.475502+0000","last_peered":"2024-09-21T09:47:45.475502+0000","last_clean":"2024-09-21T09:47:45.475502+0000","last_became_active":"2024-09-21T09:47:20.866224+0000","last_became_peered":"2024-09-21T09:47:20.866224+0000","last_unstale":"2024-09-21T09:47:45.475502+0000","last_undegraded":"2024-09-21T09:47:45.475502+0000","last_fullsized":"2024-09-21T09:47:45.475502+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469992+0000","last_change":"2024-09-21T09:47:24.061053+0000","last_active":"2024-09-21T09:47:45.469992+0000","last_peered":"2024-09-21T09:47:45.469992+0000","last_clean":"2024-09-21T09:47:45.469992+0000","last_became_active":"2024-09-21T09:47:24.060632+0000","last_became_peered":"2024-09-21T09:47:24.060632+0000","last_unstale":"2024-09-21T09:47:45.469992+0000","last_undegraded":"2024-09-21T09:47:45.469992+0000","last_fullsized":"2024-09-21T09:47:45.469992+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473227+0000","last_change":"2024-09-21T09:47:20.869192+0000","last_active":"2024-09-21T09:47:45.473227+0000","last_peered":"2024-09-21T09:47:45.473227+0000","last_clean":"2024-09-21T09:47:45.473227+0000","last_became_active":"2024-09-21T09:47:20.868800+0000","last_became_peered":"2024-09-21T09:47:20.868800+0000","last_unstale":"2024-09-21T09:47:45.473227+0000","last_undegraded":"2024-09-21T09:47:45.473227+0000","last_fullsized":"2024-09-21T09:47:45.473227+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473014+0000","last_change":"2024-09-21T09:47:24.056658+0000","last_active":"2024-09-21T09:47:45.473014+0000","last_peered":"2024-09-21T09:47:45.473014+0000","last_clean":"2024-09-21T09:47:45.473014+0000","last_became_active":"2024-09-21T09:47:24.056366+0000","last_became_peered":"2024-09-21T09:47:24.056366+0000","last_unstale":"2024-09-21T09:47:45.473014+0000","last_undegraded":"2024-09-21T09:47:45.473014+0000","last_fullsized":"2024-09-21T09:47:45.473014+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469527+0000","last_change":"2024-09-21T09:47:20.861899+0000","last_active":"2024-09-21T09:47:45.469527+0000","last_peered":"2024-09-21T09:47:45.469527+0000","last_clean":"2024-09-21T09:47:45.469527+0000","last_became_active":"2024-09-21T09:47:20.861684+0000","last_became_peered":"2024-09-21T09:47:20.861684+0000","last_unstale":"2024-09-21T09:47:45.469527+0000","last_undegraded":"2024-09-21T09:47:45.469527+0000","last_fullsized":"2024-09-21T09:47:45.469527+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475106+0000","last_change":"2024-09-21T09:47:24.058235+0000","last_active":"2024-09-21T09:47:45.475106+0000","last_peered":"2024-09-21T09:47:45.475106+0000","last_clean":"2024-09-21T09:47:45.475106+0000","last_became_active":"2024-09-21T09:47:24.057972+0000","last_became_peered":"2024-09-21T09:47:24.057972+0000","last_unstale":"2024-09-21T09:47:45.475106+0000","last_undegraded":"2024-09-21T09:47:45.475106+0000","last_fullsized":"2024-09-21T09:47:45.475106+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475569+0000","last_change":"2024-09-21T09:47:20.861547+0000","last_active":"2024-09-21T09:47:45.475569+0000","last_peered":"2024-09-21T09:47:45.475569+0000","last_clean":"2024-09-21T09:47:45.475569+0000","last_became_active":"2024-09-21T09:47:20.861254+0000","last_became_peered":"2024-09-21T09:47:20.861254+0000","last_unstale":"2024-09-21T09:47:45.475569+0000","last_undegraded":"2024-09-21T09:47:45.475569+0000","last_fullsized":"2024-09-21T09:47:45.475569+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471048+0000","last_change":"2024-09-21T09:47:24.060191+0000","last_active":"2024-09-21T09:47:45.471048+0000","last_peered":"2024-09-21T09:47:45.471048+0000","last_clean":"2024-09-21T09:47:45.471048+0000","last_became_active":"2024-09-21T09:47:24.059929+0000","last_became_peered":"2024-09-21T09:47:24.059929+0000","last_unstale":"2024-09-21T09:47:45.471048+0000","last_undegraded":"2024-09-21T09:47:45.471048+0000","last_fullsized":"2024-09-21T09:47:45.471048+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475637+0000","last_change":"2024-09-21T09:47:20.862181+0000","last_active":"2024-09-21T09:47:45.475637+0000","last_peered":"2024-09-21T09:47:45.475637+0000","last_clean":"2024-09-21T09:47:45.475637+0000","last_became_active":"2024-09-21T09:47:20.861903+0000","last_became_peered":"2024-09-21T09:47:20.861903+0000","last_unstale":"2024-09-21T09:47:45.475637+0000","last_undegraded":"2024-09-21T09:47:45.475637+0000","last_fullsized":"2024-09-21T09:47:45.475637+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474023+0000","last_change":"2024-09-21T09:47:24.058849+0000","last_active":"2024-09-21T09:47:45.474023+0000","last_peered":"2024-09-21T09:47:45.474023+0000","last_clean":"2024-09-21T09:47:45.474023+0000","last_became_active":"2024-09-21T09:47:24.058564+0000","last_became_peered":"2024-09-21T09:47:24.058564+0000","last_unstale":"2024-09-21T09:47:45.474023+0000","last_undegraded":"2024-09-21T09:47:45.474023+0000","last_fullsized":"2024-09-21T09:47:45.474023+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476182+0000","last_change":"2024-09-21T09:47:20.866885+0000","last_active":"2024-09-21T09:47:45.476182+0000","last_peered":"2024-09-21T09:47:45.476182+0000","last_clean":"2024-09-21T09:47:45.476182+0000","last_became_active":"2024-09-21T09:47:20.866436+0000","last_became_peered":"2024-09-21T09:47:20.866436+0000","last_unstale":"2024-09-21T09:47:45.476182+0000","last_undegraded":"2024-09-21T09:47:45.476182+0000","last_fullsized":"2024-09-21T09:47:45.476182+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475549+0000","last_change":"2024-09-21T09:47:24.060367+0000","last_active":"2024-09-21T09:47:45.475549+0000","last_peered":"2024-09-21T09:47:45.475549+0000","last_clean":"2024-09-21T09:47:45.475549+0000","last_became_active":"2024-09-21T09:47:24.060083+0000","last_became_peered":"2024-09-21T09:47:24.060083+0000","last_unstale":"2024-09-21T09:47:45.475549+0000","last_undegraded":"2024-09-21T09:47:45.475549+0000","last_fullsized":"2024-09-21T09:47:45.475549+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475765+0000","last_change":"2024-09-21T09:47:20.858947+0000","last_active":"2024-09-21T09:47:45.475765+0000","last_peered":"2024-09-21T09:47:45.475765+0000","last_clean":"2024-09-21T09:47:45.475765+0000","last_became_active":"2024-09-21T09:47:20.858680+0000","last_became_peered":"2024-09-21T09:47:20.858680+0000","last_unstale":"2024-09-21T09:47:45.475765+0000","last_undegraded":"2024-09-21T09:47:45.475765+0000","last_fullsized":"2024-09-21T09:47:45.475765+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470043+0000","last_change":"2024-09-21T09:47:24.060498+0000","last_active":"2024-09-21T09:47:45.470043+0000","last_peered":"2024-09-21T09:47:45.470043+0000","last_clean":"2024-09-21T09:47:45.470043+0000","last_became_active":"2024-09-21T09:47:24.060179+0000","last_became_peered":"2024-09-21T09:47:24.060179+0000","last_unstale":"2024-09-21T09:47:45.470043+0000","last_undegraded":"2024-09-21T09:47:45.470043+0000","last_fullsized":"2024-09-21T09:47:45.470043+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475113+0000","last_change":"2024-09-21T09:47:20.865797+0000","last_active":"2024-09-21T09:47:45.475113+0000","last_peered":"2024-09-21T09:47:45.475113+0000","last_clean":"2024-09-21T09:47:45.475113+0000","last_became_active":"2024-09-21T09:47:20.865466+0000","last_became_peered":"2024-09-21T09:47:20.865466+0000","last_unstale":"2024-09-21T09:47:45.475113+0000","last_undegraded":"2024-09-21T09:47:45.475113+0000","last_fullsized":"2024-09-21T09:47:45.475113+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473171+0000","last_change":"2024-09-21T09:47:24.058690+0000","last_active":"2024-09-21T09:47:45.473171+0000","last_peered":"2024-09-21T09:47:45.473171+0000","last_clean":"2024-09-21T09:47:45.473171+0000","last_became_active":"2024-09-21T09:47:24.058371+0000","last_became_peered":"2024-09-21T09:47:24.058371+0000","last_unstale":"2024-09-21T09:47:45.473171+0000","last_undegraded":"2024-09-21T09:47:45.473171+0000","last_fullsized":"2024-09-21T09:47:45.473171+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473115+0000","last_change":"2024-09-21T09:47:20.868994+0000","last_active":"2024-09-21T09:47:45.473115+0000","last_peered":"2024-09-21T09:47:45.473115+0000","last_clean":"2024-09-21T09:47:45.473115+0000","last_became_active":"2024-09-21T09:47:20.868686+0000","last_became_peered":"2024-09-21T09:47:20.868686+0000","last_unstale":"2024-09-21T09:47:45.473115+0000","last_undegraded":"2024-09-21T09:47:45.473115+0000","last_fullsized":"2024-09-21T09:47:45.473115+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470397+0000","last_change":"2024-09-21T09:47:24.055068+0000","last_active":"2024-09-21T09:47:45.470397+0000","last_peered":"2024-09-21T09:47:45.470397+0000","last_clean":"2024-09-21T09:47:45.470397+0000","last_became_active":"2024-09-21T09:47:24.054785+0000","last_became_peered":"2024-09-21T09:47:24.054785+0000","last_unstale":"2024-09-21T09:47:45.470397+0000","last_undegraded":"2024-09-21T09:47:45.470397+0000","last_fullsized":"2024-09-21T09:47:45.470397+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475383+0000","last_change":"2024-09-21T09:47:20.862674+0000","last_active":"2024-09-21T09:47:45.475383+0000","last_peered":"2024-09-21T09:47:45.475383+0000","last_clean":"2024-09-21T09:47:45.475383+0000","last_became_active":"2024-09-21T09:47:20.861988+0000","last_became_peered":"2024-09-21T09:47:20.861988+0000","last_unstale":"2024-09-21T09:47:45.475383+0000","last_undegraded":"2024-09-21T09:47:45.475383+0000","last_fullsized":"2024-09-21T09:47:45.475383+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469755+0000","last_change":"2024-09-21T09:47:24.058826+0000","last_active":"2024-09-21T09:47:45.469755+0000","last_peered":"2024-09-21T09:47:45.469755+0000","last_clean":"2024-09-21T09:47:45.469755+0000","last_became_active":"2024-09-21T09:47:24.058565+0000","last_became_peered":"2024-09-21T09:47:24.058565+0000","last_unstale":"2024-09-21T09:47:45.469755+0000","last_undegraded":"2024-09-21T09:47:45.469755+0000","last_fullsized":"2024-09-21T09:47:45.469755+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475408+0000","last_change":"2024-09-21T09:47:20.861142+0000","last_active":"2024-09-21T09:47:45.475408+0000","last_peered":"2024-09-21T09:47:45.475408+0000","last_clean":"2024-09-21T09:47:45.475408+0000","last_became_active":"2024-09-21T09:47:20.860916+0000","last_became_peered":"2024-09-21T09:47:20.860916+0000","last_unstale":"2024-09-21T09:47:45.475408+0000","last_undegraded":"2024-09-21T09:47:45.475408+0000","last_fullsized":"2024-09-21T09:47:45.475408+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.481406+0000","last_change":"2024-09-21T09:47:46.481406+0000","last_active":"2024-09-21T09:47:46.481406+0000","last_peered":"2024-09-21T09:47:46.481406+0000","last_clean":"2024-09-21T09:47:46.481406+0000","last_became_active":"2024-09-21T09:47:46.481006+0000","last_became_peered":"2024-09-21T09:47:46.481006+0000","last_unstale":"2024-09-21T09:47:46.481406+0000","last_undegraded":"2024-09-21T09:47:46.481406+0000","last_fullsized":"2024-09-21T09:47:46.481406+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:18:56.780478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475414+0000","last_change":"2024-09-21T09:47:20.865234+0000","last_active":"2024-09-21T09:47:45.475414+0000","last_peered":"2024-09-21T09:47:45.475414+0000","last_clean":"2024-09-21T09:47:45.475414+0000","last_became_active":"2024-09-21T09:47:20.863790+0000","last_became_peered":"2024-09-21T09:47:20.863790+0000","last_unstale":"2024-09-21T09:47:45.475414+0000","last_undegraded":"2024-09-21T09:47:45.475414+0000","last_fullsized":"2024-09-21T09:47:45.475414+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474777+0000","last_change":"2024-09-21T09:47:24.056216+0000","last_active":"2024-09-21T09:47:45.474777+0000","last_peered":"2024-09-21T09:47:45.474777+0000","last_clean":"2024-09-21T09:47:45.474777+0000","last_became_active":"2024-09-21T09:47:24.055928+0000","last_became_peered":"2024-09-21T09:47:24.055928+0000","last_unstale":"2024-09-21T09:47:45.474777+0000","last_undegraded":"2024-09-21T09:47:45.474777+0000","last_fullsized":"2024-09-21T09:47:45.474777+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473935+0000","last_change":"2024-09-21T09:47:24.063533+0000","last_active":"2024-09-21T09:47:45.473935+0000","last_peered":"2024-09-21T09:47:45.473935+0000","last_clean":"2024-09-21T09:47:45.473935+0000","last_became_active":"2024-09-21T09:47:24.063293+0000","last_became_peered":"2024-09-21T09:47:24.063293+0000","last_unstale":"2024-09-21T09:47:45.473935+0000","last_undegraded":"2024-09-21T09:47:45.473935+0000","last_fullsized":"2024-09-21T09:47:45.473935+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474478+0000","last_change":"2024-09-21T09:47:20.864537+0000","last_active":"2024-09-21T09:47:45.474478+0000","last_peered":"2024-09-21T09:47:45.474478+0000","last_clean":"2024-09-21T09:47:45.474478+0000","last_became_active":"2024-09-21T09:47:20.864212+0000","last_became_peered":"2024-09-21T09:47:20.864212+0000","last_unstale":"2024-09-21T09:47:45.474478+0000","last_undegraded":"2024-09-21T09:47:45.474478+0000","last_fullsized":"2024-09-21T09:47:45.474478+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774197+0000","last_change":"2024-09-21T09:47:46.774197+0000","last_active":"2024-09-21T09:47:46.774197+0000","last_peered":"2024-09-21T09:47:46.774197+0000","last_clean":"2024-09-21T09:47:46.774197+0000","last_became_active":"2024-09-21T09:47:46.773846+0000","last_became_peered":"2024-09-21T09:47:46.773846+0000","last_unstale":"2024-09-21T09:47:46.774197+0000","last_undegraded":"2024-09-21T09:47:46.774197+0000","last_fullsized":"2024-09-21T09:47:46.774197+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475812+0000","last_change":"2024-09-21T09:47:20.866709+0000","last_active":"2024-09-21T09:47:45.475812+0000","last_peered":"2024-09-21T09:47:45.475812+0000","last_clean":"2024-09-21T09:47:45.475812+0000","last_became_active":"2024-09-21T09:47:20.866026+0000","last_became_peered":"2024-09-21T09:47:20.866026+0000","last_unstale":"2024-09-21T09:47:45.475812+0000","last_undegraded":"2024-09-21T09:47:45.475812+0000","last_fullsized":"2024-09-21T09:47:45.475812+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473620+0000","last_change":"2024-09-21T09:47:24.060801+0000","last_active":"2024-09-21T09:47:45.473620+0000","last_peered":"2024-09-21T09:47:45.473620+0000","last_clean":"2024-09-21T09:47:45.473620+0000","last_became_active":"2024-09-21T09:47:24.060536+0000","last_became_peered":"2024-09-21T09:47:24.060536+0000","last_unstale":"2024-09-21T09:47:45.473620+0000","last_undegraded":"2024-09-21T09:47:45.473620+0000","last_fullsized":"2024-09-21T09:47:45.473620+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470652+0000","last_change":"2024-09-21T09:47:20.864529+0000","last_active":"2024-09-21T09:47:45.470652+0000","last_peered":"2024-09-21T09:47:45.470652+0000","last_clean":"2024-09-21T09:47:45.470652+0000","last_became_active":"2024-09-21T09:47:20.864185+0000","last_became_peered":"2024-09-21T09:47:20.864185+0000","last_unstale":"2024-09-21T09:47:45.470652+0000","last_undegraded":"2024-09-21T09:47:45.470652+0000","last_fullsized":"2024-09-21T09:47:45.470652+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470571+0000","last_change":"2024-09-21T09:47:24.065677+0000","last_active":"2024-09-21T09:47:45.470571+0000","last_peered":"2024-09-21T09:47:45.470571+0000","last_clean":"2024-09-21T09:47:45.470571+0000","last_became_active":"2024-09-21T09:47:24.065373+0000","last_became_peered":"2024-09-21T09:47:24.065373+0000","last_unstale":"2024-09-21T09:47:45.470571+0000","last_undegraded":"2024-09-21T09:47:45.470571+0000","last_fullsized":"2024-09-21T09:47:45.470571+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473774+0000","last_change":"2024-09-21T09:47:20.865947+0000","last_active":"2024-09-21T09:47:45.473774+0000","last_peered":"2024-09-21T09:47:45.473774+0000","last_clean":"2024-09-21T09:47:45.473774+0000","last_became_active":"2024-09-21T09:47:20.865683+0000","last_became_peered":"2024-09-21T09:47:20.865683+0000","last_unstale":"2024-09-21T09:47:45.473774+0000","last_undegraded":"2024-09-21T09:47:45.473774+0000","last_fullsized":"2024-09-21T09:47:45.473774+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430223,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298524,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446628,"statfs":{"total":95995035648,"available":95689347072,"internally_reserved":0,"allocated":1191936,"data_stored":841969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67100000000000004}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.62}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52900000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54700000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58999999999999997}]}]},{"osd":6,"up_from":42,"seq":180388626451,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298512,"kb_used_data":1160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446640,"statfs":{"total":95995035648,"available":95689359360,"internally_reserved":0,"allocated":1187840,"data_stored":842370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.498},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.57199999999999995}]},{"osd":1,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.626},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.61299999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.32000000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.41599999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.439}]}]},{"osd":5,"up_from":36,"seq":154618822680,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":588,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":602112,"data_stored":252336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.57799999999999996}]},{"osd":1,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.625},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.60799999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.64900000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.55400000000000005}]},{"osd":4,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.71299999999999997}]},{"osd":6,"last update":"Sat Sep 21 09:47:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.72799999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.755}]}]},{"osd":4,"up_from":30,"seq":128849018909,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":606208,"data_stored":251705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.56299999999999994,"15min":0.57299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68999999999999995,"5min":0.748,"15min":0.748},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.432,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.55300000000000005}]},{"osd":1,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.38,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.65600000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.435,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67700000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.53700000000000003}]},{"osd":2,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.71599999999999997,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.51900000000000002,"15min":0.51600000000000001},"min":{"1min":0.42399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.624,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.59499999999999997}]},{"osd":3,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.51800000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.64200000000000002,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.44900000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.68300000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.379}]},{"osd":6,"last update":"Sat Sep 21 09:47:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.42999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39300000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182434,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297952,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447200,"statfs":{"total":95995035648,"available":95689932800,"internally_reserved":0,"allocated":606208,"data_stored":252350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.39800000000000002,"15min":0.40000000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.55500000000000005,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.253},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41299999999999998,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.59499999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.26400000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44500000000000001,"15min":0.45000000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42799999999999999,"15min":0.43099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.69299999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65600000000000003,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.307}]},{"osd":4,"last update":"Sat Sep 21 09:47:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.58399999999999996,"15min":0.57999999999999996},"min":{"1min":0.41799999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86899999999999999,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.86199999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.85299999999999998,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.65100000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.71799999999999997}]},{"osd":6,"last update":"Sat Sep 21 09:47:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.76500000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69799999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411366,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":589824,"data_stored":251009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.35999999999999999,"15min":0.35399999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.27300000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.57799999999999996,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59999999999999998,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.312},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.442,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63800000000000001,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.379}]},{"osd":3,"last update":"Sat Sep 21 09:47:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.46800000000000003,"15min":0.47499999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.56200000000000006,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63900000000000001,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.33000000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:47:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73099999999999998,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.75900000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.44}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.56799999999999995}]},{"osd":6,"last update":"Sat Sep 21 09:47:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.49399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51900000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574891,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":598016,"data_stored":251169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.34799999999999998,"15min":0.32400000000000001},"min":{"1min":0.24399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.52500000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.379,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.28199999999999997,"5min":0.214,"15min":0.214},"max":{"1min":0.47899999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.375,"15min":0.375},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.49399999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.41099999999999998,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.63500000000000001,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.28599999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:47:22 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57299999999999995,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.309},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.65000000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.442}]},{"osd":4,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71199999999999997,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.67600000000000005,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.74399999999999999}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.68899999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":0,"up_from":8,"seq":34359738416,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298528,"kb_used_data":1168,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446624,"statfs":{"total":95995035648,"available":95689342976,"internally_reserved":0,"allocated":1196032,"data_stored":841778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:47:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.33800000000000002,"15min":0.316},"min":{"1min":0.24199999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.374,"15min":0.34999999999999998},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Sep 21 09:47:01 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.27500000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.59099999999999997,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.317},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.39500000000000002,"15min":0.38500000000000001},"min":{"1min":0.33300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.248}]},{"osd":3,"last update":"Sat Sep 21 09:47:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.317},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.33100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.33200000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:47:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.56000000000000005,"15min":0.55200000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.629,"5min":0.57799999999999996,"15min":0.56999999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.502}]},{"osd":5,"last update":"Sat Sep 21 09:47:06 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.65600000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.46300000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.628}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:48:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:00 smithi105 ceph-mon[31286]: from='client.14517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:48:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:00 smithi105 ceph-mon[31286]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:00 smithi033 ceph-mon[39045]: from='client.14517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:48:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:00 smithi033 ceph-mon[39045]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:00 smithi033 ceph-mon[27969]: from='client.14517 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:48:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:00 smithi033 ceph-mon[27969]: pgmap v206: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:02 smithi105 ceph-mon[31286]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:02 smithi033 ceph-mon[27969]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:02 smithi033 ceph-mon[39045]: pgmap v207: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:03.136 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:48:03.402 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:04.776 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:04 smithi033 ceph-mon[39045]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:04.776 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:04 smithi033 ceph-mon[27969]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:04.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:04 smithi105 ceph-mon[31286]: pgmap v208: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:05.252 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:48:05.253 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-21T09:48:05.867 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":208,"stamp":"2024-09-21T09:48:03.387350+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":146,"num_write_kb":1641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111,"ondisk_log_size":111,"up":291,"acting":291,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":291,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385284,"kb_used_data":6424,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575932,"statfs":{"total":767960285184,"available":765517754368,"internally_reserved":0,"allocated":6578176,"data_stored":3784686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003616"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473431+0000","last_change":"2024-09-21T09:47:24.066961+0000","last_active":"2024-09-21T09:47:45.473431+0000","last_peered":"2024-09-21T09:47:45.473431+0000","last_clean":"2024-09-21T09:47:45.473431+0000","last_became_active":"2024-09-21T09:47:24.066672+0000","last_became_peered":"2024-09-21T09:47:24.066672+0000","last_unstale":"2024-09-21T09:47:45.473431+0000","last_undegraded":"2024-09-21T09:47:45.473431+0000","last_fullsized":"2024-09-21T09:47:45.473431+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475762+0000","last_change":"2024-09-21T09:47:24.059165+0000","last_active":"2024-09-21T09:47:45.475762+0000","last_peered":"2024-09-21T09:47:45.475762+0000","last_clean":"2024-09-21T09:47:45.475762+0000","last_became_active":"2024-09-21T09:47:24.058702+0000","last_became_peered":"2024-09-21T09:47:24.058702+0000","last_unstale":"2024-09-21T09:47:45.475762+0000","last_undegraded":"2024-09-21T09:47:45.475762+0000","last_fullsized":"2024-09-21T09:47:45.475762+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.472960+0000","last_change":"2024-09-21T09:47:24.060359+0000","last_active":"2024-09-21T09:47:45.472960+0000","last_peered":"2024-09-21T09:47:45.472960+0000","last_clean":"2024-09-21T09:47:45.472960+0000","last_became_active":"2024-09-21T09:47:24.060088+0000","last_became_peered":"2024-09-21T09:47:24.060088+0000","last_unstale":"2024-09-21T09:47:45.472960+0000","last_undegraded":"2024-09-21T09:47:45.472960+0000","last_fullsized":"2024-09-21T09:47:45.472960+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473799+0000","last_change":"2024-09-21T09:47:24.065047+0000","last_active":"2024-09-21T09:47:45.473799+0000","last_peered":"2024-09-21T09:47:45.473799+0000","last_clean":"2024-09-21T09:47:45.473799+0000","last_became_active":"2024-09-21T09:47:24.064743+0000","last_became_peered":"2024-09-21T09:47:24.064743+0000","last_unstale":"2024-09-21T09:47:45.473799+0000","last_undegraded":"2024-09-21T09:47:45.473799+0000","last_fullsized":"2024-09-21T09:47:45.473799+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475055+0000","last_change":"2024-09-21T09:47:24.069692+0000","last_active":"2024-09-21T09:47:45.475055+0000","last_peered":"2024-09-21T09:47:45.475055+0000","last_clean":"2024-09-21T09:47:45.475055+0000","last_became_active":"2024-09-21T09:47:24.069090+0000","last_became_peered":"2024-09-21T09:47:24.069090+0000","last_unstale":"2024-09-21T09:47:45.475055+0000","last_undegraded":"2024-09-21T09:47:45.475055+0000","last_fullsized":"2024-09-21T09:47:45.475055+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475319+0000","last_change":"2024-09-21T09:47:24.053540+0000","last_active":"2024-09-21T09:47:45.475319+0000","last_peered":"2024-09-21T09:47:45.475319+0000","last_clean":"2024-09-21T09:47:45.475319+0000","last_became_active":"2024-09-21T09:47:24.053163+0000","last_became_peered":"2024-09-21T09:47:24.053163+0000","last_unstale":"2024-09-21T09:47:45.475319+0000","last_undegraded":"2024-09-21T09:47:45.475319+0000","last_fullsized":"2024-09-21T09:47:45.475319+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774760+0000","last_change":"2024-09-21T09:47:46.774760+0000","last_active":"2024-09-21T09:47:46.774760+0000","last_peered":"2024-09-21T09:47:46.774760+0000","last_clean":"2024-09-21T09:47:46.774760+0000","last_became_active":"2024-09-21T09:47:46.774445+0000","last_became_peered":"2024-09-21T09:47:46.774445+0000","last_unstale":"2024-09-21T09:47:46.774760+0000","last_undegraded":"2024-09-21T09:47:46.774760+0000","last_fullsized":"2024-09-21T09:47:46.774760+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469675+0000","last_change":"2024-09-21T09:47:24.064091+0000","last_active":"2024-09-21T09:47:45.469675+0000","last_peered":"2024-09-21T09:47:45.469675+0000","last_clean":"2024-09-21T09:47:45.469675+0000","last_became_active":"2024-09-21T09:47:24.063698+0000","last_became_peered":"2024-09-21T09:47:24.063698+0000","last_unstale":"2024-09-21T09:47:45.469675+0000","last_undegraded":"2024-09-21T09:47:45.469675+0000","last_fullsized":"2024-09-21T09:47:45.469675+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476570+0000","last_change":"2024-09-21T09:47:24.071747+0000","last_active":"2024-09-21T09:47:45.476570+0000","last_peered":"2024-09-21T09:47:45.476570+0000","last_clean":"2024-09-21T09:47:45.476570+0000","last_became_active":"2024-09-21T09:47:24.071475+0000","last_became_peered":"2024-09-21T09:47:24.071475+0000","last_unstale":"2024-09-21T09:47:45.476570+0000","last_undegraded":"2024-09-21T09:47:45.476570+0000","last_fullsized":"2024-09-21T09:47:45.476570+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475068+0000","last_change":"2024-09-21T09:47:24.058216+0000","last_active":"2024-09-21T09:47:45.475068+0000","last_peered":"2024-09-21T09:47:45.475068+0000","last_clean":"2024-09-21T09:47:45.475068+0000","last_became_active":"2024-09-21T09:47:24.057940+0000","last_became_peered":"2024-09-21T09:47:24.057940+0000","last_unstale":"2024-09-21T09:47:45.475068+0000","last_undegraded":"2024-09-21T09:47:45.475068+0000","last_fullsized":"2024-09-21T09:47:45.475068+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474927+0000","last_change":"2024-09-21T09:47:24.053543+0000","last_active":"2024-09-21T09:47:45.474927+0000","last_peered":"2024-09-21T09:47:45.474927+0000","last_clean":"2024-09-21T09:47:45.474927+0000","last_became_active":"2024-09-21T09:47:24.053264+0000","last_became_peered":"2024-09-21T09:47:24.053264+0000","last_unstale":"2024-09-21T09:47:45.474927+0000","last_undegraded":"2024-09-21T09:47:45.474927+0000","last_fullsized":"2024-09-21T09:47:45.474927+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.850968+0000","last_change":"2024-09-21T09:47:46.850968+0000","last_active":"2024-09-21T09:47:46.850968+0000","last_peered":"2024-09-21T09:47:46.850968+0000","last_clean":"2024-09-21T09:47:46.850968+0000","last_became_active":"2024-09-21T09:47:46.850604+0000","last_became_peered":"2024-09-21T09:47:46.850604+0000","last_unstale":"2024-09-21T09:47:46.850968+0000","last_undegraded":"2024-09-21T09:47:46.850968+0000","last_fullsized":"2024-09-21T09:47:46.850968+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470142+0000","last_change":"2024-09-21T09:47:24.064334+0000","last_active":"2024-09-21T09:47:45.470142+0000","last_peered":"2024-09-21T09:47:45.470142+0000","last_clean":"2024-09-21T09:47:45.470142+0000","last_became_active":"2024-09-21T09:47:24.064063+0000","last_became_peered":"2024-09-21T09:47:24.064063+0000","last_unstale":"2024-09-21T09:47:45.470142+0000","last_undegraded":"2024-09-21T09:47:45.470142+0000","last_fullsized":"2024-09-21T09:47:45.470142+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473158+0000","last_change":"2024-09-21T09:47:24.058842+0000","last_active":"2024-09-21T09:47:45.473158+0000","last_peered":"2024-09-21T09:47:45.473158+0000","last_clean":"2024-09-21T09:47:45.473158+0000","last_became_active":"2024-09-21T09:47:24.058245+0000","last_became_peered":"2024-09-21T09:47:24.058245+0000","last_unstale":"2024-09-21T09:47:45.473158+0000","last_undegraded":"2024-09-21T09:47:45.473158+0000","last_fullsized":"2024-09-21T09:47:45.473158+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475221+0000","last_change":"2024-09-21T09:47:24.052028+0000","last_active":"2024-09-21T09:47:45.475221+0000","last_peered":"2024-09-21T09:47:45.475221+0000","last_clean":"2024-09-21T09:47:45.475221+0000","last_became_active":"2024-09-21T09:47:24.051766+0000","last_became_peered":"2024-09-21T09:47:24.051766+0000","last_unstale":"2024-09-21T09:47:45.475221+0000","last_undegraded":"2024-09-21T09:47:45.475221+0000","last_fullsized":"2024-09-21T09:47:45.475221+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473412+0000","last_change":"2024-09-21T09:47:24.060117+0000","last_active":"2024-09-21T09:47:45.473412+0000","last_peered":"2024-09-21T09:47:45.473412+0000","last_clean":"2024-09-21T09:47:45.473412+0000","last_became_active":"2024-09-21T09:47:24.059852+0000","last_became_peered":"2024-09-21T09:47:24.059852+0000","last_unstale":"2024-09-21T09:47:45.473412+0000","last_undegraded":"2024-09-21T09:47:45.473412+0000","last_fullsized":"2024-09-21T09:47:45.473412+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.480400+0000","last_change":"2024-09-21T09:47:46.480400+0000","last_active":"2024-09-21T09:47:46.480400+0000","last_peered":"2024-09-21T09:47:46.480400+0000","last_clean":"2024-09-21T09:47:46.480400+0000","last_became_active":"2024-09-21T09:47:46.480103+0000","last_became_peered":"2024-09-21T09:47:46.480103+0000","last_unstale":"2024-09-21T09:47:46.480400+0000","last_undegraded":"2024-09-21T09:47:46.480400+0000","last_fullsized":"2024-09-21T09:47:46.480400+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469780+0000","last_change":"2024-09-21T09:47:24.057147+0000","last_active":"2024-09-21T09:47:45.469780+0000","last_peered":"2024-09-21T09:47:45.469780+0000","last_clean":"2024-09-21T09:47:45.469780+0000","last_became_active":"2024-09-21T09:47:24.056798+0000","last_became_peered":"2024-09-21T09:47:24.056798+0000","last_unstale":"2024-09-21T09:47:45.469780+0000","last_undegraded":"2024-09-21T09:47:45.469780+0000","last_fullsized":"2024-09-21T09:47:45.469780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475925+0000","last_change":"2024-09-21T09:47:24.070640+0000","last_active":"2024-09-21T09:47:45.475925+0000","last_peered":"2024-09-21T09:47:45.475925+0000","last_clean":"2024-09-21T09:47:45.475925+0000","last_became_active":"2024-09-21T09:47:24.070359+0000","last_became_peered":"2024-09-21T09:47:24.070359+0000","last_unstale":"2024-09-21T09:47:45.475925+0000","last_undegraded":"2024-09-21T09:47:45.475925+0000","last_fullsized":"2024-09-21T09:47:45.475925+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474605+0000","last_change":"2024-09-21T09:47:24.056659+0000","last_active":"2024-09-21T09:47:45.474605+0000","last_peered":"2024-09-21T09:47:45.474605+0000","last_clean":"2024-09-21T09:47:45.474605+0000","last_became_active":"2024-09-21T09:47:24.056360+0000","last_became_peered":"2024-09-21T09:47:24.056360+0000","last_unstale":"2024-09-21T09:47:45.474605+0000","last_undegraded":"2024-09-21T09:47:45.474605+0000","last_fullsized":"2024-09-21T09:47:45.474605+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473450+0000","last_change":"2024-09-21T09:47:24.065154+0000","last_active":"2024-09-21T09:47:45.473450+0000","last_peered":"2024-09-21T09:47:45.473450+0000","last_clean":"2024-09-21T09:47:45.473450+0000","last_became_active":"2024-09-21T09:47:24.064231+0000","last_became_peered":"2024-09-21T09:47:24.064231+0000","last_unstale":"2024-09-21T09:47:45.473450+0000","last_undegraded":"2024-09-21T09:47:45.473450+0000","last_fullsized":"2024-09-21T09:47:45.473450+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475326+0000","last_change":"2024-09-21T09:47:24.062716+0000","last_active":"2024-09-21T09:47:45.475326+0000","last_peered":"2024-09-21T09:47:45.475326+0000","last_clean":"2024-09-21T09:47:45.475326+0000","last_became_active":"2024-09-21T09:47:24.062437+0000","last_became_peered":"2024-09-21T09:47:24.062437+0000","last_unstale":"2024-09-21T09:47:45.475326+0000","last_undegraded":"2024-09-21T09:47:45.475326+0000","last_fullsized":"2024-09-21T09:47:45.475326+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470046+0000","last_change":"2024-09-21T09:47:24.051387+0000","last_active":"2024-09-21T09:47:45.470046+0000","last_peered":"2024-09-21T09:47:45.470046+0000","last_clean":"2024-09-21T09:47:45.470046+0000","last_became_active":"2024-09-21T09:47:24.051090+0000","last_became_peered":"2024-09-21T09:47:24.051090+0000","last_unstale":"2024-09-21T09:47:45.470046+0000","last_undegraded":"2024-09-21T09:47:45.470046+0000","last_fullsized":"2024-09-21T09:47:45.470046+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473500+0000","last_change":"2024-09-21T09:47:24.058079+0000","last_active":"2024-09-21T09:47:45.473500+0000","last_peered":"2024-09-21T09:47:45.473500+0000","last_clean":"2024-09-21T09:47:45.473500+0000","last_became_active":"2024-09-21T09:47:24.057753+0000","last_became_peered":"2024-09-21T09:47:24.057753+0000","last_unstale":"2024-09-21T09:47:45.473500+0000","last_undegraded":"2024-09-21T09:47:45.473500+0000","last_fullsized":"2024-09-21T09:47:45.473500+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469933+0000","last_change":"2024-09-21T09:47:24.057220+0000","last_active":"2024-09-21T09:47:45.469933+0000","last_peered":"2024-09-21T09:47:45.469933+0000","last_clean":"2024-09-21T09:47:45.469933+0000","last_became_active":"2024-09-21T09:47:24.056915+0000","last_became_peered":"2024-09-21T09:47:24.056915+0000","last_unstale":"2024-09-21T09:47:45.469933+0000","last_undegraded":"2024-09-21T09:47:45.469933+0000","last_fullsized":"2024-09-21T09:47:45.469933+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475187+0000","last_change":"2024-09-21T09:47:24.057310+0000","last_active":"2024-09-21T09:47:45.475187+0000","last_peered":"2024-09-21T09:47:45.475187+0000","last_clean":"2024-09-21T09:47:45.475187+0000","last_became_active":"2024-09-21T09:47:24.057047+0000","last_became_peered":"2024-09-21T09:47:24.057047+0000","last_unstale":"2024-09-21T09:47:45.475187+0000","last_undegraded":"2024-09-21T09:47:45.475187+0000","last_fullsized":"2024-09-21T09:47:45.475187+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475460+0000","last_change":"2024-09-21T09:47:24.060976+0000","last_active":"2024-09-21T09:47:45.475460+0000","last_peered":"2024-09-21T09:47:45.475460+0000","last_clean":"2024-09-21T09:47:45.475460+0000","last_became_active":"2024-09-21T09:47:24.060725+0000","last_became_peered":"2024-09-21T09:47:24.060725+0000","last_unstale":"2024-09-21T09:47:45.475460+0000","last_undegraded":"2024-09-21T09:47:45.475460+0000","last_fullsized":"2024-09-21T09:47:45.475460+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470271+0000","last_change":"2024-09-21T09:47:24.053563+0000","last_active":"2024-09-21T09:47:45.470271+0000","last_peered":"2024-09-21T09:47:45.470271+0000","last_clean":"2024-09-21T09:47:45.470271+0000","last_became_active":"2024-09-21T09:47:24.052853+0000","last_became_peered":"2024-09-21T09:47:24.052853+0000","last_unstale":"2024-09-21T09:47:45.470271+0000","last_undegraded":"2024-09-21T09:47:45.470271+0000","last_fullsized":"2024-09-21T09:47:45.470271+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473532+0000","last_change":"2024-09-21T09:47:24.068204+0000","last_active":"2024-09-21T09:47:45.473532+0000","last_peered":"2024-09-21T09:47:45.473532+0000","last_clean":"2024-09-21T09:47:45.473532+0000","last_became_active":"2024-09-21T09:47:24.067911+0000","last_became_peered":"2024-09-21T09:47:24.067911+0000","last_unstale":"2024-09-21T09:47:45.473532+0000","last_undegraded":"2024-09-21T09:47:45.473532+0000","last_fullsized":"2024-09-21T09:47:45.473532+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475415+0000","last_change":"2024-09-21T09:47:24.058882+0000","last_active":"2024-09-21T09:47:45.475415+0000","last_peered":"2024-09-21T09:47:45.475415+0000","last_clean":"2024-09-21T09:47:45.475415+0000","last_became_active":"2024-09-21T09:47:24.058587+0000","last_became_peered":"2024-09-21T09:47:24.058587+0000","last_unstale":"2024-09-21T09:47:45.475415+0000","last_undegraded":"2024-09-21T09:47:45.475415+0000","last_fullsized":"2024-09-21T09:47:45.475415+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469392+0000","last_change":"2024-09-21T09:47:24.060446+0000","last_active":"2024-09-21T09:47:45.469392+0000","last_peered":"2024-09-21T09:47:45.469392+0000","last_clean":"2024-09-21T09:47:45.469392+0000","last_became_active":"2024-09-21T09:47:24.060165+0000","last_became_peered":"2024-09-21T09:47:24.060165+0000","last_unstale":"2024-09-21T09:47:45.469392+0000","last_undegraded":"2024-09-21T09:47:45.469392+0000","last_fullsized":"2024-09-21T09:47:45.469392+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473573+0000","last_change":"2024-09-21T09:47:24.059974+0000","last_active":"2024-09-21T09:47:45.473573+0000","last_peered":"2024-09-21T09:47:45.473573+0000","last_clean":"2024-09-21T09:47:45.473573+0000","last_became_active":"2024-09-21T09:47:24.059270+0000","last_became_peered":"2024-09-21T09:47:24.059270+0000","last_unstale":"2024-09-21T09:47:45.473573+0000","last_undegraded":"2024-09-21T09:47:45.473573+0000","last_fullsized":"2024-09-21T09:47:45.473573+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470251+0000","last_change":"2024-09-21T09:47:24.048941+0000","last_active":"2024-09-21T09:47:45.470251+0000","last_peered":"2024-09-21T09:47:45.470251+0000","last_clean":"2024-09-21T09:47:45.470251+0000","last_became_active":"2024-09-21T09:47:24.048631+0000","last_became_peered":"2024-09-21T09:47:24.048631+0000","last_unstale":"2024-09-21T09:47:45.470251+0000","last_undegraded":"2024-09-21T09:47:45.470251+0000","last_fullsized":"2024-09-21T09:47:45.470251+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470304+0000","last_change":"2024-09-21T09:47:20.866010+0000","last_active":"2024-09-21T09:47:45.470304+0000","last_peered":"2024-09-21T09:47:45.470304+0000","last_clean":"2024-09-21T09:47:45.470304+0000","last_became_active":"2024-09-21T09:47:20.865732+0000","last_became_peered":"2024-09-21T09:47:20.865732+0000","last_unstale":"2024-09-21T09:47:45.470304+0000","last_undegraded":"2024-09-21T09:47:45.470304+0000","last_fullsized":"2024-09-21T09:47:45.470304+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473645+0000","last_change":"2024-09-21T09:47:24.067813+0000","last_active":"2024-09-21T09:47:45.473645+0000","last_peered":"2024-09-21T09:47:45.473645+0000","last_clean":"2024-09-21T09:47:45.473645+0000","last_became_active":"2024-09-21T09:47:24.067412+0000","last_became_peered":"2024-09-21T09:47:24.067412+0000","last_unstale":"2024-09-21T09:47:45.473645+0000","last_undegraded":"2024-09-21T09:47:45.473645+0000","last_fullsized":"2024-09-21T09:47:45.473645+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":27,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.481320+0000","last_change":"2024-09-21T09:47:46.481320+0000","last_active":"2024-09-21T09:47:46.481320+0000","last_peered":"2024-09-21T09:47:46.481320+0000","last_clean":"2024-09-21T09:47:46.481320+0000","last_became_active":"2024-09-21T09:47:46.480905+0000","last_became_peered":"2024-09-21T09:47:46.480905+0000","last_unstale":"2024-09-21T09:47:46.481320+0000","last_undegraded":"2024-09-21T09:47:46.481320+0000","last_fullsized":"2024-09-21T09:47:46.481320+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:23.831301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470210+0000","last_change":"2024-09-21T09:47:20.859829+0000","last_active":"2024-09-21T09:47:45.470210+0000","last_peered":"2024-09-21T09:47:45.470210+0000","last_clean":"2024-09-21T09:47:45.470210+0000","last_became_active":"2024-09-21T09:47:20.859507+0000","last_became_peered":"2024-09-21T09:47:20.859507+0000","last_unstale":"2024-09-21T09:47:45.470210+0000","last_undegraded":"2024-09-21T09:47:45.470210+0000","last_fullsized":"2024-09-21T09:47:45.470210+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469715+0000","last_change":"2024-09-21T09:47:24.053702+0000","last_active":"2024-09-21T09:47:45.469715+0000","last_peered":"2024-09-21T09:47:45.469715+0000","last_clean":"2024-09-21T09:47:45.469715+0000","last_became_active":"2024-09-21T09:47:24.053295+0000","last_became_peered":"2024-09-21T09:47:24.053295+0000","last_unstale":"2024-09-21T09:47:45.469715+0000","last_undegraded":"2024-09-21T09:47:45.469715+0000","last_fullsized":"2024-09-21T09:47:45.469715+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475673+0000","last_change":"2024-09-21T09:47:20.863181+0000","last_active":"2024-09-21T09:47:45.475673+0000","last_peered":"2024-09-21T09:47:45.475673+0000","last_clean":"2024-09-21T09:47:45.475673+0000","last_became_active":"2024-09-21T09:47:20.862895+0000","last_became_peered":"2024-09-21T09:47:20.862895+0000","last_unstale":"2024-09-21T09:47:45.475673+0000","last_undegraded":"2024-09-21T09:47:45.475673+0000","last_fullsized":"2024-09-21T09:47:45.475673+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475132+0000","last_change":"2024-09-21T09:47:24.051248+0000","last_active":"2024-09-21T09:47:45.475132+0000","last_peered":"2024-09-21T09:47:45.475132+0000","last_clean":"2024-09-21T09:47:45.475132+0000","last_became_active":"2024-09-21T09:47:24.050918+0000","last_became_peered":"2024-09-21T09:47:24.050918+0000","last_unstale":"2024-09-21T09:47:45.475132+0000","last_undegraded":"2024-09-21T09:47:45.475132+0000","last_fullsized":"2024-09-21T09:47:45.475132+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475023+0000","last_change":"2024-09-21T09:47:20.865532+0000","last_active":"2024-09-21T09:47:45.475023+0000","last_peered":"2024-09-21T09:47:45.475023+0000","last_clean":"2024-09-21T09:47:45.475023+0000","last_became_active":"2024-09-21T09:47:20.865103+0000","last_became_peered":"2024-09-21T09:47:20.865103+0000","last_unstale":"2024-09-21T09:47:45.475023+0000","last_undegraded":"2024-09-21T09:47:45.475023+0000","last_fullsized":"2024-09-21T09:47:45.475023+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474292+0000","last_change":"2024-09-21T09:47:24.067330+0000","last_active":"2024-09-21T09:47:45.474292+0000","last_peered":"2024-09-21T09:47:45.474292+0000","last_clean":"2024-09-21T09:47:45.474292+0000","last_became_active":"2024-09-21T09:47:24.066939+0000","last_became_peered":"2024-09-21T09:47:24.066939+0000","last_unstale":"2024-09-21T09:47:45.474292+0000","last_undegraded":"2024-09-21T09:47:45.474292+0000","last_fullsized":"2024-09-21T09:47:45.474292+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474434+0000","last_change":"2024-09-21T09:47:20.860605+0000","last_active":"2024-09-21T09:47:45.474434+0000","last_peered":"2024-09-21T09:47:45.474434+0000","last_clean":"2024-09-21T09:47:45.474434+0000","last_became_active":"2024-09-21T09:47:20.860288+0000","last_became_peered":"2024-09-21T09:47:20.860288+0000","last_unstale":"2024-09-21T09:47:45.474434+0000","last_undegraded":"2024-09-21T09:47:45.474434+0000","last_fullsized":"2024-09-21T09:47:45.474434+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473649+0000","last_change":"2024-09-21T09:47:24.057860+0000","last_active":"2024-09-21T09:47:45.473649+0000","last_peered":"2024-09-21T09:47:45.473649+0000","last_clean":"2024-09-21T09:47:45.473649+0000","last_became_active":"2024-09-21T09:47:24.057594+0000","last_became_peered":"2024-09-21T09:47:24.057594+0000","last_unstale":"2024-09-21T09:47:45.473649+0000","last_undegraded":"2024-09-21T09:47:45.473649+0000","last_fullsized":"2024-09-21T09:47:45.473649+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475431+0000","last_change":"2024-09-21T09:47:20.857026+0000","last_active":"2024-09-21T09:47:45.475431+0000","last_peered":"2024-09-21T09:47:45.475431+0000","last_clean":"2024-09-21T09:47:45.475431+0000","last_became_active":"2024-09-21T09:47:20.856719+0000","last_became_peered":"2024-09-21T09:47:20.856719+0000","last_unstale":"2024-09-21T09:47:45.475431+0000","last_undegraded":"2024-09-21T09:47:45.475431+0000","last_fullsized":"2024-09-21T09:47:45.475431+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471636+0000","last_change":"2024-09-21T09:47:24.059537+0000","last_active":"2024-09-21T09:47:45.471636+0000","last_peered":"2024-09-21T09:47:45.471636+0000","last_clean":"2024-09-21T09:47:45.471636+0000","last_became_active":"2024-09-21T09:47:24.059264+0000","last_became_peered":"2024-09-21T09:47:24.059264+0000","last_unstale":"2024-09-21T09:47:45.471636+0000","last_undegraded":"2024-09-21T09:47:45.471636+0000","last_fullsized":"2024-09-21T09:47:45.471636+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"57'6","reported_seq":22,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474711+0000","last_change":"2024-09-21T09:47:20.864446+0000","last_active":"2024-09-21T09:47:45.474711+0000","last_peered":"2024-09-21T09:47:45.474711+0000","last_clean":"2024-09-21T09:47:45.474711+0000","last_became_active":"2024-09-21T09:47:20.864160+0000","last_became_peered":"2024-09-21T09:47:20.864160+0000","last_unstale":"2024-09-21T09:47:45.474711+0000","last_undegraded":"2024-09-21T09:47:45.474711+0000","last_fullsized":"2024-09-21T09:47:45.474711+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475529+0000","last_change":"2024-09-21T09:47:24.070010+0000","last_active":"2024-09-21T09:47:45.475529+0000","last_peered":"2024-09-21T09:47:45.475529+0000","last_clean":"2024-09-21T09:47:45.475529+0000","last_became_active":"2024-09-21T09:47:24.069698+0000","last_became_peered":"2024-09-21T09:47:24.069698+0000","last_unstale":"2024-09-21T09:47:45.475529+0000","last_undegraded":"2024-09-21T09:47:45.475529+0000","last_fullsized":"2024-09-21T09:47:45.475529+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473662+0000","last_change":"2024-09-21T09:47:20.858242+0000","last_active":"2024-09-21T09:47:45.473662+0000","last_peered":"2024-09-21T09:47:45.473662+0000","last_clean":"2024-09-21T09:47:45.473662+0000","last_became_active":"2024-09-21T09:47:20.857951+0000","last_became_peered":"2024-09-21T09:47:20.857951+0000","last_unstale":"2024-09-21T09:47:45.473662+0000","last_undegraded":"2024-09-21T09:47:45.473662+0000","last_fullsized":"2024-09-21T09:47:45.473662+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473418+0000","last_change":"2024-09-21T09:47:24.063666+0000","last_active":"2024-09-21T09:47:45.473418+0000","last_peered":"2024-09-21T09:47:45.473418+0000","last_clean":"2024-09-21T09:47:45.473418+0000","last_became_active":"2024-09-21T09:47:24.063116+0000","last_became_peered":"2024-09-21T09:47:24.063116+0000","last_unstale":"2024-09-21T09:47:45.473418+0000","last_undegraded":"2024-09-21T09:47:45.473418+0000","last_fullsized":"2024-09-21T09:47:45.473418+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475534+0000","last_change":"2024-09-21T09:47:20.867134+0000","last_active":"2024-09-21T09:47:45.475534+0000","last_peered":"2024-09-21T09:47:45.475534+0000","last_clean":"2024-09-21T09:47:45.475534+0000","last_became_active":"2024-09-21T09:47:20.866875+0000","last_became_peered":"2024-09-21T09:47:20.866875+0000","last_unstale":"2024-09-21T09:47:45.475534+0000","last_undegraded":"2024-09-21T09:47:45.475534+0000","last_fullsized":"2024-09-21T09:47:45.475534+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475144+0000","last_change":"2024-09-21T09:47:24.069831+0000","last_active":"2024-09-21T09:47:45.475144+0000","last_peered":"2024-09-21T09:47:45.475144+0000","last_clean":"2024-09-21T09:47:45.475144+0000","last_became_active":"2024-09-21T09:47:24.069569+0000","last_became_peered":"2024-09-21T09:47:24.069569+0000","last_unstale":"2024-09-21T09:47:45.475144+0000","last_undegraded":"2024-09-21T09:47:45.475144+0000","last_fullsized":"2024-09-21T09:47:45.475144+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474104+0000","last_change":"2024-09-21T09:47:20.862992+0000","last_active":"2024-09-21T09:47:45.474104+0000","last_peered":"2024-09-21T09:47:45.474104+0000","last_clean":"2024-09-21T09:47:45.474104+0000","last_became_active":"2024-09-21T09:47:20.862582+0000","last_became_peered":"2024-09-21T09:47:20.862582+0000","last_unstale":"2024-09-21T09:47:45.474104+0000","last_undegraded":"2024-09-21T09:47:45.474104+0000","last_fullsized":"2024-09-21T09:47:45.474104+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473661+0000","last_change":"2024-09-21T09:47:24.058311+0000","last_active":"2024-09-21T09:47:45.473661+0000","last_peered":"2024-09-21T09:47:45.473661+0000","last_clean":"2024-09-21T09:47:45.473661+0000","last_became_active":"2024-09-21T09:47:24.057961+0000","last_became_peered":"2024-09-21T09:47:24.057961+0000","last_unstale":"2024-09-21T09:47:45.473661+0000","last_undegraded":"2024-09-21T09:47:45.473661+0000","last_fullsized":"2024-09-21T09:47:45.473661+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":24,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469194+0000","last_change":"2024-09-21T09:47:20.856509+0000","last_active":"2024-09-21T09:47:45.469194+0000","last_peered":"2024-09-21T09:47:45.469194+0000","last_clean":"2024-09-21T09:47:45.469194+0000","last_became_active":"2024-09-21T09:47:20.856229+0000","last_became_peered":"2024-09-21T09:47:20.856229+0000","last_unstale":"2024-09-21T09:47:45.469194+0000","last_undegraded":"2024-09-21T09:47:45.469194+0000","last_fullsized":"2024-09-21T09:47:45.469194+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474200+0000","last_change":"2024-09-21T09:47:24.069301+0000","last_active":"2024-09-21T09:47:45.474200+0000","last_peered":"2024-09-21T09:47:45.474200+0000","last_clean":"2024-09-21T09:47:45.474200+0000","last_became_active":"2024-09-21T09:47:24.068917+0000","last_became_peered":"2024-09-21T09:47:24.068917+0000","last_unstale":"2024-09-21T09:47:45.474200+0000","last_undegraded":"2024-09-21T09:47:45.474200+0000","last_fullsized":"2024-09-21T09:47:45.474200+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471246+0000","last_change":"2024-09-21T09:47:20.862118+0000","last_active":"2024-09-21T09:47:45.471246+0000","last_peered":"2024-09-21T09:47:45.471246+0000","last_clean":"2024-09-21T09:47:45.471246+0000","last_became_active":"2024-09-21T09:47:20.861815+0000","last_became_peered":"2024-09-21T09:47:20.861815+0000","last_unstale":"2024-09-21T09:47:45.471246+0000","last_undegraded":"2024-09-21T09:47:45.471246+0000","last_fullsized":"2024-09-21T09:47:45.471246+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474780+0000","last_change":"2024-09-21T09:47:24.069502+0000","last_active":"2024-09-21T09:47:45.474780+0000","last_peered":"2024-09-21T09:47:45.474780+0000","last_clean":"2024-09-21T09:47:45.474780+0000","last_became_active":"2024-09-21T09:47:24.069235+0000","last_became_peered":"2024-09-21T09:47:24.069235+0000","last_unstale":"2024-09-21T09:47:45.474780+0000","last_undegraded":"2024-09-21T09:47:45.474780+0000","last_fullsized":"2024-09-21T09:47:45.474780+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469921+0000","last_change":"2024-09-21T09:47:20.862503+0000","last_active":"2024-09-21T09:47:45.469921+0000","last_peered":"2024-09-21T09:47:45.469921+0000","last_clean":"2024-09-21T09:47:45.469921+0000","last_became_active":"2024-09-21T09:47:20.862237+0000","last_became_peered":"2024-09-21T09:47:20.862237+0000","last_unstale":"2024-09-21T09:47:45.469921+0000","last_undegraded":"2024-09-21T09:47:45.469921+0000","last_fullsized":"2024-09-21T09:47:45.469921+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476257+0000","last_change":"2024-09-21T09:47:24.069273+0000","last_active":"2024-09-21T09:47:45.476257+0000","last_peered":"2024-09-21T09:47:45.476257+0000","last_clean":"2024-09-21T09:47:45.476257+0000","last_became_active":"2024-09-21T09:47:24.068990+0000","last_became_peered":"2024-09-21T09:47:24.068990+0000","last_unstale":"2024-09-21T09:47:45.476257+0000","last_undegraded":"2024-09-21T09:47:45.476257+0000","last_fullsized":"2024-09-21T09:47:45.476257+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475558+0000","last_change":"2024-09-21T09:47:20.858095+0000","last_active":"2024-09-21T09:47:45.475558+0000","last_peered":"2024-09-21T09:47:45.475558+0000","last_clean":"2024-09-21T09:47:45.475558+0000","last_became_active":"2024-09-21T09:47:20.857760+0000","last_became_peered":"2024-09-21T09:47:20.857760+0000","last_unstale":"2024-09-21T09:47:45.475558+0000","last_undegraded":"2024-09-21T09:47:45.475558+0000","last_fullsized":"2024-09-21T09:47:45.475558+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473915+0000","last_change":"2024-09-21T09:47:24.063342+0000","last_active":"2024-09-21T09:47:45.473915+0000","last_peered":"2024-09-21T09:47:45.473915+0000","last_clean":"2024-09-21T09:47:45.473915+0000","last_became_active":"2024-09-21T09:47:24.063070+0000","last_became_peered":"2024-09-21T09:47:24.063070+0000","last_unstale":"2024-09-21T09:47:45.473915+0000","last_undegraded":"2024-09-21T09:47:45.473915+0000","last_fullsized":"2024-09-21T09:47:45.473915+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469526+0000","last_change":"2024-09-21T09:47:20.859702+0000","last_active":"2024-09-21T09:47:45.469526+0000","last_peered":"2024-09-21T09:47:45.469526+0000","last_clean":"2024-09-21T09:47:45.469526+0000","last_became_active":"2024-09-21T09:47:20.859443+0000","last_became_peered":"2024-09-21T09:47:20.859443+0000","last_unstale":"2024-09-21T09:47:45.469526+0000","last_undegraded":"2024-09-21T09:47:45.469526+0000","last_fullsized":"2024-09-21T09:47:45.469526+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475617+0000","last_change":"2024-09-21T09:47:24.070338+0000","last_active":"2024-09-21T09:47:45.475617+0000","last_peered":"2024-09-21T09:47:45.475617+0000","last_clean":"2024-09-21T09:47:45.475617+0000","last_became_active":"2024-09-21T09:47:24.070074+0000","last_became_peered":"2024-09-21T09:47:24.070074+0000","last_unstale":"2024-09-21T09:47:45.475617+0000","last_undegraded":"2024-09-21T09:47:45.475617+0000","last_fullsized":"2024-09-21T09:47:45.475617+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"22'72","reported_seq":25,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473341+0000","last_change":"2024-09-21T09:46:57.426595+0000","last_active":"2024-09-21T09:47:45.473341+0000","last_peered":"2024-09-21T09:47:45.473341+0000","last_clean":"2024-09-21T09:47:45.473341+0000","last_became_active":"2024-09-21T09:46:57.418997+0000","last_became_peered":"2024-09-21T09:46:57.418997+0000","last_unstale":"2024-09-21T09:47:45.473341+0000","last_undegraded":"2024-09-21T09:47:45.473341+0000","last_fullsized":"2024-09-21T09:47:45.473341+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_clean_scrub_stamp":"2024-09-21T09:44:57.819716+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475502+0000","last_change":"2024-09-21T09:47:20.866500+0000","last_active":"2024-09-21T09:47:45.475502+0000","last_peered":"2024-09-21T09:47:45.475502+0000","last_clean":"2024-09-21T09:47:45.475502+0000","last_became_active":"2024-09-21T09:47:20.866224+0000","last_became_peered":"2024-09-21T09:47:20.866224+0000","last_unstale":"2024-09-21T09:47:45.475502+0000","last_undegraded":"2024-09-21T09:47:45.475502+0000","last_fullsized":"2024-09-21T09:47:45.475502+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469992+0000","last_change":"2024-09-21T09:47:24.061053+0000","last_active":"2024-09-21T09:47:45.469992+0000","last_peered":"2024-09-21T09:47:45.469992+0000","last_clean":"2024-09-21T09:47:45.469992+0000","last_became_active":"2024-09-21T09:47:24.060632+0000","last_became_peered":"2024-09-21T09:47:24.060632+0000","last_unstale":"2024-09-21T09:47:45.469992+0000","last_undegraded":"2024-09-21T09:47:45.469992+0000","last_fullsized":"2024-09-21T09:47:45.469992+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473227+0000","last_change":"2024-09-21T09:47:20.869192+0000","last_active":"2024-09-21T09:47:45.473227+0000","last_peered":"2024-09-21T09:47:45.473227+0000","last_clean":"2024-09-21T09:47:45.473227+0000","last_became_active":"2024-09-21T09:47:20.868800+0000","last_became_peered":"2024-09-21T09:47:20.868800+0000","last_unstale":"2024-09-21T09:47:45.473227+0000","last_undegraded":"2024-09-21T09:47:45.473227+0000","last_fullsized":"2024-09-21T09:47:45.473227+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473014+0000","last_change":"2024-09-21T09:47:24.056658+0000","last_active":"2024-09-21T09:47:45.473014+0000","last_peered":"2024-09-21T09:47:45.473014+0000","last_clean":"2024-09-21T09:47:45.473014+0000","last_became_active":"2024-09-21T09:47:24.056366+0000","last_became_peered":"2024-09-21T09:47:24.056366+0000","last_unstale":"2024-09-21T09:47:45.473014+0000","last_undegraded":"2024-09-21T09:47:45.473014+0000","last_fullsized":"2024-09-21T09:47:45.473014+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469527+0000","last_change":"2024-09-21T09:47:20.861899+0000","last_active":"2024-09-21T09:47:45.469527+0000","last_peered":"2024-09-21T09:47:45.469527+0000","last_clean":"2024-09-21T09:47:45.469527+0000","last_became_active":"2024-09-21T09:47:20.861684+0000","last_became_peered":"2024-09-21T09:47:20.861684+0000","last_unstale":"2024-09-21T09:47:45.469527+0000","last_undegraded":"2024-09-21T09:47:45.469527+0000","last_fullsized":"2024-09-21T09:47:45.469527+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475106+0000","last_change":"2024-09-21T09:47:24.058235+0000","last_active":"2024-09-21T09:47:45.475106+0000","last_peered":"2024-09-21T09:47:45.475106+0000","last_clean":"2024-09-21T09:47:45.475106+0000","last_became_active":"2024-09-21T09:47:24.057972+0000","last_became_peered":"2024-09-21T09:47:24.057972+0000","last_unstale":"2024-09-21T09:47:45.475106+0000","last_undegraded":"2024-09-21T09:47:45.475106+0000","last_fullsized":"2024-09-21T09:47:45.475106+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475569+0000","last_change":"2024-09-21T09:47:20.861547+0000","last_active":"2024-09-21T09:47:45.475569+0000","last_peered":"2024-09-21T09:47:45.475569+0000","last_clean":"2024-09-21T09:47:45.475569+0000","last_became_active":"2024-09-21T09:47:20.861254+0000","last_became_peered":"2024-09-21T09:47:20.861254+0000","last_unstale":"2024-09-21T09:47:45.475569+0000","last_undegraded":"2024-09-21T09:47:45.475569+0000","last_fullsized":"2024-09-21T09:47:45.475569+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.471048+0000","last_change":"2024-09-21T09:47:24.060191+0000","last_active":"2024-09-21T09:47:45.471048+0000","last_peered":"2024-09-21T09:47:45.471048+0000","last_clean":"2024-09-21T09:47:45.471048+0000","last_became_active":"2024-09-21T09:47:24.059929+0000","last_became_peered":"2024-09-21T09:47:24.059929+0000","last_unstale":"2024-09-21T09:47:45.471048+0000","last_undegraded":"2024-09-21T09:47:45.471048+0000","last_fullsized":"2024-09-21T09:47:45.471048+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475637+0000","last_change":"2024-09-21T09:47:20.862181+0000","last_active":"2024-09-21T09:47:45.475637+0000","last_peered":"2024-09-21T09:47:45.475637+0000","last_clean":"2024-09-21T09:47:45.475637+0000","last_became_active":"2024-09-21T09:47:20.861903+0000","last_became_peered":"2024-09-21T09:47:20.861903+0000","last_unstale":"2024-09-21T09:47:45.475637+0000","last_undegraded":"2024-09-21T09:47:45.475637+0000","last_fullsized":"2024-09-21T09:47:45.475637+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474023+0000","last_change":"2024-09-21T09:47:24.058849+0000","last_active":"2024-09-21T09:47:45.474023+0000","last_peered":"2024-09-21T09:47:45.474023+0000","last_clean":"2024-09-21T09:47:45.474023+0000","last_became_active":"2024-09-21T09:47:24.058564+0000","last_became_peered":"2024-09-21T09:47:24.058564+0000","last_unstale":"2024-09-21T09:47:45.474023+0000","last_undegraded":"2024-09-21T09:47:45.474023+0000","last_fullsized":"2024-09-21T09:47:45.474023+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.476182+0000","last_change":"2024-09-21T09:47:20.866885+0000","last_active":"2024-09-21T09:47:45.476182+0000","last_peered":"2024-09-21T09:47:45.476182+0000","last_clean":"2024-09-21T09:47:45.476182+0000","last_became_active":"2024-09-21T09:47:20.866436+0000","last_became_peered":"2024-09-21T09:47:20.866436+0000","last_unstale":"2024-09-21T09:47:45.476182+0000","last_undegraded":"2024-09-21T09:47:45.476182+0000","last_fullsized":"2024-09-21T09:47:45.476182+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475549+0000","last_change":"2024-09-21T09:47:24.060367+0000","last_active":"2024-09-21T09:47:45.475549+0000","last_peered":"2024-09-21T09:47:45.475549+0000","last_clean":"2024-09-21T09:47:45.475549+0000","last_became_active":"2024-09-21T09:47:24.060083+0000","last_became_peered":"2024-09-21T09:47:24.060083+0000","last_unstale":"2024-09-21T09:47:45.475549+0000","last_undegraded":"2024-09-21T09:47:45.475549+0000","last_fullsized":"2024-09-21T09:47:45.475549+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475765+0000","last_change":"2024-09-21T09:47:20.858947+0000","last_active":"2024-09-21T09:47:45.475765+0000","last_peered":"2024-09-21T09:47:45.475765+0000","last_clean":"2024-09-21T09:47:45.475765+0000","last_became_active":"2024-09-21T09:47:20.858680+0000","last_became_peered":"2024-09-21T09:47:20.858680+0000","last_unstale":"2024-09-21T09:47:45.475765+0000","last_undegraded":"2024-09-21T09:47:45.475765+0000","last_fullsized":"2024-09-21T09:47:45.475765+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470043+0000","last_change":"2024-09-21T09:47:24.060498+0000","last_active":"2024-09-21T09:47:45.470043+0000","last_peered":"2024-09-21T09:47:45.470043+0000","last_clean":"2024-09-21T09:47:45.470043+0000","last_became_active":"2024-09-21T09:47:24.060179+0000","last_became_peered":"2024-09-21T09:47:24.060179+0000","last_unstale":"2024-09-21T09:47:45.470043+0000","last_undegraded":"2024-09-21T09:47:45.470043+0000","last_fullsized":"2024-09-21T09:47:45.470043+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475113+0000","last_change":"2024-09-21T09:47:20.865797+0000","last_active":"2024-09-21T09:47:45.475113+0000","last_peered":"2024-09-21T09:47:45.475113+0000","last_clean":"2024-09-21T09:47:45.475113+0000","last_became_active":"2024-09-21T09:47:20.865466+0000","last_became_peered":"2024-09-21T09:47:20.865466+0000","last_unstale":"2024-09-21T09:47:45.475113+0000","last_undegraded":"2024-09-21T09:47:45.475113+0000","last_fullsized":"2024-09-21T09:47:45.475113+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473171+0000","last_change":"2024-09-21T09:47:24.058690+0000","last_active":"2024-09-21T09:47:45.473171+0000","last_peered":"2024-09-21T09:47:45.473171+0000","last_clean":"2024-09-21T09:47:45.473171+0000","last_became_active":"2024-09-21T09:47:24.058371+0000","last_became_peered":"2024-09-21T09:47:24.058371+0000","last_unstale":"2024-09-21T09:47:45.473171+0000","last_undegraded":"2024-09-21T09:47:45.473171+0000","last_fullsized":"2024-09-21T09:47:45.473171+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473115+0000","last_change":"2024-09-21T09:47:20.868994+0000","last_active":"2024-09-21T09:47:45.473115+0000","last_peered":"2024-09-21T09:47:45.473115+0000","last_clean":"2024-09-21T09:47:45.473115+0000","last_became_active":"2024-09-21T09:47:20.868686+0000","last_became_peered":"2024-09-21T09:47:20.868686+0000","last_unstale":"2024-09-21T09:47:45.473115+0000","last_undegraded":"2024-09-21T09:47:45.473115+0000","last_fullsized":"2024-09-21T09:47:45.473115+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470397+0000","last_change":"2024-09-21T09:47:24.055068+0000","last_active":"2024-09-21T09:47:45.470397+0000","last_peered":"2024-09-21T09:47:45.470397+0000","last_clean":"2024-09-21T09:47:45.470397+0000","last_became_active":"2024-09-21T09:47:24.054785+0000","last_became_peered":"2024-09-21T09:47:24.054785+0000","last_unstale":"2024-09-21T09:47:45.470397+0000","last_undegraded":"2024-09-21T09:47:45.470397+0000","last_fullsized":"2024-09-21T09:47:45.470397+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475383+0000","last_change":"2024-09-21T09:47:20.862674+0000","last_active":"2024-09-21T09:47:45.475383+0000","last_peered":"2024-09-21T09:47:45.475383+0000","last_clean":"2024-09-21T09:47:45.475383+0000","last_became_active":"2024-09-21T09:47:20.861988+0000","last_became_peered":"2024-09-21T09:47:20.861988+0000","last_unstale":"2024-09-21T09:47:45.475383+0000","last_undegraded":"2024-09-21T09:47:45.475383+0000","last_fullsized":"2024-09-21T09:47:45.475383+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.469755+0000","last_change":"2024-09-21T09:47:24.058826+0000","last_active":"2024-09-21T09:47:45.469755+0000","last_peered":"2024-09-21T09:47:45.469755+0000","last_clean":"2024-09-21T09:47:45.469755+0000","last_became_active":"2024-09-21T09:47:24.058565+0000","last_became_peered":"2024-09-21T09:47:24.058565+0000","last_unstale":"2024-09-21T09:47:45.469755+0000","last_undegraded":"2024-09-21T09:47:45.469755+0000","last_fullsized":"2024-09-21T09:47:45.469755+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":16,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475408+0000","last_change":"2024-09-21T09:47:20.861142+0000","last_active":"2024-09-21T09:47:45.475408+0000","last_peered":"2024-09-21T09:47:45.475408+0000","last_clean":"2024-09-21T09:47:45.475408+0000","last_became_active":"2024-09-21T09:47:20.860916+0000","last_became_peered":"2024-09-21T09:47:20.860916+0000","last_unstale":"2024-09-21T09:47:45.475408+0000","last_undegraded":"2024-09-21T09:47:45.475408+0000","last_fullsized":"2024-09-21T09:47:45.475408+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":23,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.481406+0000","last_change":"2024-09-21T09:47:46.481406+0000","last_active":"2024-09-21T09:47:46.481406+0000","last_peered":"2024-09-21T09:47:46.481406+0000","last_clean":"2024-09-21T09:47:46.481406+0000","last_became_active":"2024-09-21T09:47:46.481006+0000","last_became_peered":"2024-09-21T09:47:46.481006+0000","last_unstale":"2024-09-21T09:47:46.481406+0000","last_undegraded":"2024-09-21T09:47:46.481406+0000","last_fullsized":"2024-09-21T09:47:46.481406+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:18:56.780478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475414+0000","last_change":"2024-09-21T09:47:20.865234+0000","last_active":"2024-09-21T09:47:45.475414+0000","last_peered":"2024-09-21T09:47:45.475414+0000","last_clean":"2024-09-21T09:47:45.475414+0000","last_became_active":"2024-09-21T09:47:20.863790+0000","last_became_peered":"2024-09-21T09:47:20.863790+0000","last_unstale":"2024-09-21T09:47:45.475414+0000","last_undegraded":"2024-09-21T09:47:45.475414+0000","last_fullsized":"2024-09-21T09:47:45.475414+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474777+0000","last_change":"2024-09-21T09:47:24.056216+0000","last_active":"2024-09-21T09:47:45.474777+0000","last_peered":"2024-09-21T09:47:45.474777+0000","last_clean":"2024-09-21T09:47:45.474777+0000","last_became_active":"2024-09-21T09:47:24.055928+0000","last_became_peered":"2024-09-21T09:47:24.055928+0000","last_unstale":"2024-09-21T09:47:45.474777+0000","last_undegraded":"2024-09-21T09:47:45.474777+0000","last_fullsized":"2024-09-21T09:47:45.474777+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473935+0000","last_change":"2024-09-21T09:47:24.063533+0000","last_active":"2024-09-21T09:47:45.473935+0000","last_peered":"2024-09-21T09:47:45.473935+0000","last_clean":"2024-09-21T09:47:45.473935+0000","last_became_active":"2024-09-21T09:47:24.063293+0000","last_became_peered":"2024-09-21T09:47:24.063293+0000","last_unstale":"2024-09-21T09:47:45.473935+0000","last_undegraded":"2024-09-21T09:47:45.473935+0000","last_fullsized":"2024-09-21T09:47:45.473935+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.474478+0000","last_change":"2024-09-21T09:47:20.864537+0000","last_active":"2024-09-21T09:47:45.474478+0000","last_peered":"2024-09-21T09:47:45.474478+0000","last_clean":"2024-09-21T09:47:45.474478+0000","last_became_active":"2024-09-21T09:47:20.864212+0000","last_became_peered":"2024-09-21T09:47:20.864212+0000","last_unstale":"2024-09-21T09:47:45.474478+0000","last_undegraded":"2024-09-21T09:47:45.474478+0000","last_fullsized":"2024-09-21T09:47:45.474478+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":10,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-09-21T09:47:46.774197+0000","last_change":"2024-09-21T09:47:46.774197+0000","last_active":"2024-09-21T09:47:46.774197+0000","last_peered":"2024-09-21T09:47:46.774197+0000","last_clean":"2024-09-21T09:47:46.774197+0000","last_became_active":"2024-09-21T09:47:46.773846+0000","last_became_peered":"2024-09-21T09:47:46.773846+0000","last_unstale":"2024-09-21T09:47:46.774197+0000","last_undegraded":"2024-09-21T09:47:46.774197+0000","last_fullsized":"2024-09-21T09:47:46.774197+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":18,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.475812+0000","last_change":"2024-09-21T09:47:20.866709+0000","last_active":"2024-09-21T09:47:45.475812+0000","last_peered":"2024-09-21T09:47:45.475812+0000","last_clean":"2024-09-21T09:47:45.475812+0000","last_became_active":"2024-09-21T09:47:20.866026+0000","last_became_peered":"2024-09-21T09:47:20.866026+0000","last_unstale":"2024-09-21T09:47:45.475812+0000","last_undegraded":"2024-09-21T09:47:45.475812+0000","last_fullsized":"2024-09-21T09:47:45.475812+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473620+0000","last_change":"2024-09-21T09:47:24.060801+0000","last_active":"2024-09-21T09:47:45.473620+0000","last_peered":"2024-09-21T09:47:45.473620+0000","last_clean":"2024-09-21T09:47:45.473620+0000","last_became_active":"2024-09-21T09:47:24.060536+0000","last_became_peered":"2024-09-21T09:47:24.060536+0000","last_unstale":"2024-09-21T09:47:45.473620+0000","last_undegraded":"2024-09-21T09:47:45.473620+0000","last_fullsized":"2024-09-21T09:47:45.473620+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":17,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470652+0000","last_change":"2024-09-21T09:47:20.864529+0000","last_active":"2024-09-21T09:47:45.470652+0000","last_peered":"2024-09-21T09:47:45.470652+0000","last_clean":"2024-09-21T09:47:45.470652+0000","last_became_active":"2024-09-21T09:47:20.864185+0000","last_became_peered":"2024-09-21T09:47:20.864185+0000","last_unstale":"2024-09-21T09:47:45.470652+0000","last_undegraded":"2024-09-21T09:47:45.470652+0000","last_fullsized":"2024-09-21T09:47:45.470652+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":13,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.470571+0000","last_change":"2024-09-21T09:47:24.065677+0000","last_active":"2024-09-21T09:47:45.470571+0000","last_peered":"2024-09-21T09:47:45.470571+0000","last_clean":"2024-09-21T09:47:45.470571+0000","last_became_active":"2024-09-21T09:47:24.065373+0000","last_became_peered":"2024-09-21T09:47:24.065373+0000","last_unstale":"2024-09-21T09:47:45.470571+0000","last_undegraded":"2024-09-21T09:47:45.470571+0000","last_fullsized":"2024-09-21T09:47:45.470571+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-21T09:47:45.473774+0000","last_change":"2024-09-21T09:47:20.865947+0000","last_active":"2024-09-21T09:47:45.473774+0000","last_peered":"2024-09-21T09:47:45.473774+0000","last_clean":"2024-09-21T09:47:45.473774+0000","last_became_active":"2024-09-21T09:47:20.865683+0000","last_became_peered":"2024-09-21T09:47:20.865683+0000","last_unstale":"2024-09-21T09:47:45.473774+0000","last_undegraded":"2024-09-21T09:47:45.473774+0000","last_fullsized":"2024-09-21T09:47:45.473774+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4157440,"data_stored":4132576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":3,"acting":3,"num_store_stats":7}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430224,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298524,"kb_used_data":1164,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446628,"statfs":{"total":95995035648,"available":95689347072,"internally_reserved":0,"allocated":1191936,"data_stored":841969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.71199999999999997}]},{"osd":1,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.61399999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.67900000000000005}]},{"osd":3,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.627}]},{"osd":4,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.53900000000000003}]},{"osd":6,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.58299999999999996}]}]},{"osd":6,"up_from":42,"seq":180388626452,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298512,"kb_used_data":1160,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446640,"statfs":{"total":95995035648,"available":95689359360,"internally_reserved":0,"allocated":1187840,"data_stored":842370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.56999999999999995}]},{"osd":1,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.51400000000000001,"15min":0.51400000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.61599999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.505}]},{"osd":3,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.48899999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.33500000000000002}]},{"osd":7,"last update":"Sat Sep 21 09:47:56 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.36499999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822681,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":588,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":602112,"data_stored":252336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66000000000000003,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.442}]},{"osd":1,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.497}]},{"osd":2,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"last":0.621},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.53700000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:47:09 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.36799999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:47:36 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.51200000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018910,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297948,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447204,"statfs":{"total":95995035648,"available":95689936896,"internally_reserved":0,"allocated":606208,"data_stored":251705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.56299999999999994,"15min":0.57299999999999995},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.68999999999999995,"5min":0.748,"15min":0.748},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.432,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.61299999999999999}]},{"osd":1,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.52600000000000002,"15min":0.53000000000000003},"min":{"1min":0.38,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.65600000000000003,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53000000000000003,"15min":0.52900000000000003},"min":{"1min":0.435,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.67700000000000005,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.54700000000000004,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.374,"15min":0.374},"max":{"1min":0.71599999999999997,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.51900000000000002,"15min":0.51600000000000001},"min":{"1min":0.42399999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.624,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.64100000000000001}]},{"osd":3,"last update":"Sat Sep 21 09:47:41 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.51800000000000002,"15min":0.52000000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.64200000000000002,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.44900000000000001,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.68300000000000005,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 21 09:47:35 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.48699999999999999}]},{"osd":7,"last update":"Sat Sep 21 09:47:58 2024","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.52700000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182435,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297952,"kb_used_data":592,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447200,"statfs":{"total":95995035648,"available":95689932800,"internally_reserved":0,"allocated":606208,"data_stored":252350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.39800000000000002,"15min":0.40000000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":0.55500000000000005,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.255},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.41299999999999998,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.59499999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.436}]},{"osd":1,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.44500000000000001,"15min":0.45000000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42799999999999999,"15min":0.43099999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.69299999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.27400000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:47:24 2024","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.65600000000000003,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.44600000000000001,"15min":0.44800000000000001},"min":{"1min":0.26700000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.373}]},{"osd":4,"last update":"Sat Sep 21 09:47:47 2024","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.58399999999999996,"15min":0.57999999999999996},"min":{"1min":0.41799999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86899999999999999,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.40899999999999997,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.85299999999999998,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.626}]},{"osd":5,"last update":"Sat Sep 21 09:47:07 2024","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:47:31 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.69599999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:47:59 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.60599999999999998}]}]},{"osd":2,"up_from":18,"seq":77309411367,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297936,"kb_used_data":576,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447216,"statfs":{"total":95995035648,"available":95689949184,"internally_reserved":0,"allocated":589824,"data_stored":251009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.35999999999999999,"15min":0.35399999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.379},"min":{"1min":0.27300000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.57799999999999996,"5min":0.66000000000000003,"15min":0.66000000000000003},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.32700000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.59999999999999998,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.442,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63800000000000001,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.46300000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:47:21 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.46800000000000003,"15min":0.47499999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.56200000000000006,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.45600000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.63900000000000001,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 21 09:47:44 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.57699999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.73099999999999998,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55800000000000005,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.75900000000000001,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.57899999999999996}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.65200000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:47:33 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Sep 21 09:48:00 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.72599999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297940,"kb_used_data":584,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447212,"statfs":{"total":95995035648,"available":95689945088,"internally_reserved":0,"allocated":598016,"data_stored":251169,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.34799999999999998,"15min":0.32400000000000001},"min":{"1min":0.24399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.52500000000000002,"5min":0.59899999999999998,"15min":0.59899999999999998},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.379,"5min":0.34599999999999997,"15min":0.32400000000000001},"min":{"1min":0.28199999999999997,"5min":0.214,"15min":0.214},"max":{"1min":0.47899999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.41799999999999998}]},{"osd":2,"last update":"Sat Sep 21 09:47:00 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.375,"15min":0.375},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.49399999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.43099999999999999,"5min":0.41099999999999998,"15min":0.40799999999999997},"min":{"1min":0.29199999999999998,"5min":0.251,"15min":0.251},"max":{"1min":0.63500000000000001,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.435}]},{"osd":3,"last update":"Sat Sep 21 09:47:22 2024","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":0.44700000000000001,"15min":0.45000000000000001},"min":{"1min":0.33000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.57299999999999995,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.47599999999999998,"15min":0.47799999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.65000000000000002,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 21 09:47:43 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.71199999999999997,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.58099999999999996,"15min":0.58499999999999996},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":0.67600000000000005,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.65000000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:47:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.63200000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:47:32 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.67200000000000004}]},{"osd":7,"last update":"Sat Sep 21 09:47:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.60699999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738417,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298528,"kb_used_data":1168,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446624,"statfs":{"total":95995035648,"available":95689342976,"internally_reserved":0,"allocated":1196032,"data_stored":841778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:47:40 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.33800000000000002,"15min":0.316},"min":{"1min":0.24199999999999999,"5min":0.188,"15min":0.188},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.374,"15min":0.34999999999999998},"min":{"1min":0.27400000000000002,"5min":0.219,"15min":0.219},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:47:01 2024","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.39500000000000002,"15min":0.39000000000000001},"min":{"1min":0.27500000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.59099999999999997,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.371},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.39500000000000002,"15min":0.38500000000000001},"min":{"1min":0.33300000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:47:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.317},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.33100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.28899999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:47:45 2024","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.56000000000000005,"15min":0.55200000000000005},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.629,"5min":0.57799999999999996,"15min":0.56999999999999995},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.54900000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:47:06 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:47:34 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:47:56 2024","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.443},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":1656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":1552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:48:05.875 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:48:05.875 INFO:tasks.cephadm:Adding prometheus.a on smithi105 2024-09-21T09:48:05.875 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch apply prometheus '1;smithi105=a' 2024-09-21T09:48:06.131 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:06 smithi105 ceph-mon[31286]: from='client.14523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:48:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:06 smithi105 ceph-mon[31286]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:06 smithi033 ceph-mon[39045]: from='client.14523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:48:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:06 smithi033 ceph-mon[39045]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:06 smithi033 ceph-mon[27969]: from='client.14523 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:48:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:06 smithi033 ceph-mon[27969]: pgmap v209: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:07.935 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled prometheus update... 2024-09-21T09:48:08.664 DEBUG:teuthology.orchestra.run.smithi105:prometheus.a> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a.service 2024-09-21T09:48:08.666 INFO:tasks.cephadm:Adding node-exporter.a on smithi033 2024-09-21T09:48:08.666 INFO:tasks.cephadm:Adding node-exporter.b on smithi105 2024-09-21T09:48:08.667 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch apply node-exporter '2;smithi033=a;smithi105=b' 2024-09-21T09:48:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:08 smithi105 ceph-mon[31286]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:08 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:08.963 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:08 smithi033 ceph-mon[39045]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:08 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:08.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:08 smithi033 ceph-mon[27969]: pgmap v210: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:08.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:08.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:08 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:09 smithi105 ceph-mon[31286]: from='client.24448 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:09 smithi105 ceph-mon[31286]: Saving service prometheus spec with placement smithi105=a;count:1 2024-09-21T09:48:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:09 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[27969]: from='client.24448 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[27969]: Saving service prometheus spec with placement smithi105=a;count:1 2024-09-21T09:48:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[39045]: from='client.24448 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[39045]: Saving service prometheus spec with placement smithi105=a;count:1 2024-09-21T09:48:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:09 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:10.726 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled node-exporter update... 2024-09-21T09:48:10.727 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:10 smithi105 ceph-mon[31286]: Deploying daemon prometheus.a on smithi105 2024-09-21T09:48:10.727 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:10 smithi105 ceph-mon[31286]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:10.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:10 smithi033 ceph-mon[27969]: Deploying daemon prometheus.a on smithi105 2024-09-21T09:48:10.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:10 smithi033 ceph-mon[27969]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:10.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:10 smithi033 ceph-mon[39045]: Deploying daemon prometheus.a on smithi105 2024-09-21T09:48:10.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:10 smithi033 ceph-mon[39045]: pgmap v211: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:11.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:11 smithi105 ceph-mon[31286]: from='client.24454 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi033=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:11.986 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:11 smithi105 ceph-mon[31286]: Saving service node-exporter spec with placement smithi033=a;smithi105=b;count:2 2024-09-21T09:48:11.987 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:11 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:11.988 DEBUG:teuthology.orchestra.run.smithi033:node-exporter.a> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.a.service 2024-09-21T09:48:11.990 DEBUG:teuthology.orchestra.run.smithi105:node-exporter.b> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.b.service 2024-09-21T09:48:11.993 INFO:tasks.cephadm:Adding alertmanager.a on smithi033 2024-09-21T09:48:11.993 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch apply alertmanager '1;smithi033=a' 2024-09-21T09:48:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:11 smithi033 ceph-mon[27969]: from='client.24454 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi033=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:11 smithi033 ceph-mon[27969]: Saving service node-exporter spec with placement smithi033=a;smithi105=b;count:2 2024-09-21T09:48:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:11 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:11 smithi033 ceph-mon[39045]: from='client.24454 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi033=a;smithi105=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:11 smithi033 ceph-mon[39045]: Saving service node-exporter spec with placement smithi033=a;smithi105=b;count:2 2024-09-21T09:48:11.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:11 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:12.450 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:12.782 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:12 smithi033 ceph-mon[39045]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:12.783 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:12 smithi033 ceph-mon[27969]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:12.841 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:12 smithi105 ceph-mon[31286]: pgmap v212: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:14 smithi033 ceph-mon[39045]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:14 smithi033 ceph-mon[27969]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:14 smithi105 ceph-mon[31286]: pgmap v213: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:16.230 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled alertmanager update... 2024-09-21T09:48:16.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:16 smithi033 ceph-mon[39045]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:16.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:16 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:16.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:16 smithi033 ceph-mon[27969]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:16.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:16 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:16 smithi105 ceph-mon[31286]: pgmap v214: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:16 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:17.536 DEBUG:teuthology.orchestra.run.smithi033:alertmanager.a> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@alertmanager.a.service 2024-09-21T09:48:17.539 INFO:tasks.cephadm:Adding grafana.a on smithi105 2024-09-21T09:48:17.540 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph orch apply grafana '1;smithi105=a' 2024-09-21T09:48:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:17 smithi105 ceph-mon[31286]: from='client.24460 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi033=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:17 smithi105 ceph-mon[31286]: Saving service alertmanager spec with placement smithi033=a;count:1 2024-09-21T09:48:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:17 smithi033 ceph-mon[39045]: from='client.24460 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi033=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:17 smithi033 ceph-mon[39045]: Saving service alertmanager spec with placement smithi033=a;count:1 2024-09-21T09:48:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:17 smithi033 ceph-mon[27969]: from='client.24460 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi033=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:17 smithi033 ceph-mon[27969]: Saving service alertmanager spec with placement smithi033=a;count:1 2024-09-21T09:48:18.256 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:18 smithi033 ceph-mon[39045]: pgmap v215: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:18.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:18 smithi033 ceph-mon[27969]: pgmap v215: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:19.006 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:18 smithi105 ceph-mon[31286]: pgmap v215: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:20 smithi033 ceph-mon[39045]: pgmap v216: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:20 smithi033 ceph-mon[27969]: pgmap v216: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:21.009 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:20 smithi105 ceph-mon[31286]: pgmap v216: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:21.698 INFO:teuthology.orchestra.run.smithi105.stdout:Scheduled grafana update... 2024-09-21T09:48:22.687 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:22 smithi105 systemd[1]: Starting Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:48:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:22 smithi033 ceph-mon[39045]: pgmap v217: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:22 smithi033 ceph-mon[39045]: from='client.24466 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:22 smithi033 ceph-mon[39045]: Saving service grafana spec with placement smithi105=a;count:1 2024-09-21T09:48:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:22 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:22 smithi033 ceph-mon[27969]: pgmap v217: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:22 smithi033 ceph-mon[27969]: from='client.24466 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:22.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:22 smithi033 ceph-mon[27969]: Saving service grafana spec with placement smithi105=a;count:1 2024-09-21T09:48:22.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:22 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:22 smithi105 ceph-mon[31286]: pgmap v217: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:22 smithi105 ceph-mon[31286]: from='client.24466 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi105=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:48:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:22 smithi105 ceph-mon[31286]: Saving service grafana spec with placement smithi105=a;count:1 2024-09-21T09:48:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:22 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:23.064 DEBUG:teuthology.orchestra.run.smithi105:grafana.a> sudo journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service 2024-09-21T09:48:23.066 INFO:tasks.cephadm:Setting up client nodes... 2024-09-21T09:48:23.067 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:48:23.335 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:23.385 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 podman[61298]: 2024-09-21 09:48:23.142498799 +0000 UTC m=+0.016544481 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:48:23.385 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 podman[61298]: 2024-09-21 09:48:23.244630879 +0000 UTC m=+0.118676537 container create cb180b46fbe0e1249659c10bd6a347d382455e31fc15360ef0cdf7b21b73fb10 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:48:23.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 podman[61298]: 2024-09-21 09:48:23.497577937 +0000 UTC m=+0.371623596 container init cb180b46fbe0e1249659c10bd6a347d382455e31fc15360ef0cdf7b21b73fb10 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 podman[61298]: 2024-09-21 09:48:23.502578185 +0000 UTC m=+0.376623845 container start cb180b46fbe0e1249659c10bd6a347d382455e31fc15360ef0cdf7b21b73fb10 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.526Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.526Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.526Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.526Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.526Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.528Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.528Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.529Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.529Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.531Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.531Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.574µs 2024-09-21T09:48:23.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.531Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.531Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.531Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=16.785µs wal_replay_duration=337.235µs wbl_replay_duration=129ns total_replay_duration=371.565µs 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.532Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.532Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.532Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.547Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.538789ms db_storage=784ns remote_storage=1.028µs web_handler=237ns query_engine=663ns scrape=386.272µs scrape_sd=21.443µs notify=932ns notify_sd=2.103µs rules=13.943629ms tracing=6.137µs 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.547Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:48:23.547Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 bash[61298]: cb180b46fbe0e1249659c10bd6a347d382455e31fc15360ef0cdf7b21b73fb10 2024-09-21T09:48:23.782 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:48:23 smithi105 systemd[1]: Started Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:48:24.821 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[27969]: pgmap v218: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:24.821 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.821 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.821 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.821 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[39045]: pgmap v218: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ignoring --setuser ceph since I am not root 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ignoring --setgroup ceph since I am not root 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-mgr[28245]: -- 172.21.15.33:0/3088889193 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564855df0340 con 0x564855dfcc00 2024-09-21T09:48:24.822 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:24.818+0000 7fb5b7352ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:48:24.938 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ignoring --setuser ceph since I am not root 2024-09-21T09:48:24.939 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ignoring --setgroup ceph since I am not root 2024-09-21T09:48:24.939 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-mgr[33389]: -- 172.21.15.105:0/3171743084 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562ba12bc340 con 0x562ba12c9000 2024-09-21T09:48:24.939 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:24.816+0000 7fe795ca5ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:48:24.939 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:24.871+0000 7fe795ca5ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:48:24.939 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-mon[31286]: pgmap v218: 97 pgs: 97 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:24.940 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.940 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.940 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' 2024-09-21T09:48:24.940 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-21T09:48:25.085 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:24.879+0000 7fb5b7352ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:48:25.085 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:24 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:24.950+0000 7fb5b7352ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:48:25.085 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:25.082+0000 7fb5b7352ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:48:25.195 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:24.936+0000 7fe795ca5ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:48:25.196 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:25.058+0000 7fe795ca5ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:48:25.420 INFO:teuthology.orchestra.run.smithi033.stdout:[client.0] 2024-09-21T09:48:25.420 INFO:teuthology.orchestra.run.smithi033.stdout: key = AQBplu5mPw64GBAAklT5LeZAZNLO1Hu2+AY9Rg== 2024-09-21T09:48:25.420 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:25.229+0000 7fb5b7352ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:48:25.446 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:25.193+0000 7fe795ca5ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:48:25.677 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:25.505+0000 7fb5b7352ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:48:25.678 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:25.568+0000 7fb5b7352ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:48:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-mon[31286]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:48:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-mon[31286]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-21T09:48:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2988263223' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2988263223' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:48:25.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:25.444+0000 7fe795ca5ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:48:25.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:25.503+0000 7fe795ca5ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:48:25.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:25.662+0000 7fe795ca5ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:48:25.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:25.738+0000 7fb5b7352ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:48:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-mon[39045]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:48:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-mon[39045]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-21T09:48:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2988263223' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2988263223' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:48:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-mon[27969]: from='mgr.14150 172.21.15.33:0/2095833751' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-21T09:48:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-mon[27969]: mgrmap e16: y(active, since 5m), standbys: x 2024-09-21T09:48:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2988263223' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:25 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2988263223' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:48:26.084 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:48:26.084 DEBUG:teuthology.orchestra.run.smithi033:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-21T09:48:26.084 DEBUG:teuthology.orchestra.run.smithi033:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-21T09:48:26.123 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-21T09:48:26.386 INFO:teuthology.orchestra.run.smithi105.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.b/config 2024-09-21T09:48:26.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:26.544+0000 7fe795ca5ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:48:26.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:26.610+0000 7fe795ca5ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:48:26.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:26 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:26.635+0000 7fb5b7352ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:48:26.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:26 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:26.703+0000 7fb5b7352ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:48:27.631 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:27.336+0000 7fe795ca5ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:48:27.631 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:27.448+0000 7fe795ca5ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:48:27.631 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:27.569+0000 7fe795ca5ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:48:27.631 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:27.628+0000 7fe795ca5ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:48:27.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:27 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:27.440+0000 7fb5b7352ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:48:27.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:27 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:27.556+0000 7fb5b7352ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:48:27.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:27 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:27.681+0000 7fb5b7352ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:48:27.891 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:27.756+0000 7fe795ca5ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:48:27.891 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:27.825+0000 7fe795ca5ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:48:27.891 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:27.888+0000 7fe795ca5ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:48:28.007 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:27 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:27.744+0000 7fb5b7352ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:48:28.007 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:27 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:27.875+0000 7fb5b7352ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:48:28.007 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:27 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:27.944+0000 7fb5b7352ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:48:28.007 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:28.004+0000 7fb5b7352ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:48:28.169 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:28.026+0000 7fe795ca5ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:48:28.169 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:28.095+0000 7fe795ca5ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:48:28.169 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:28.166+0000 7fe795ca5ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:48:28.284 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:28.147+0000 7fb5b7352ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:48:28.314 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:28.214+0000 7fb5b7352ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:48:28.314 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:28.281+0000 7fb5b7352ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:48:28.453 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:28.255+0000 7fe795ca5ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:48:28.453 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:28.316+0000 7fe795ca5ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:48:28.453 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:28.377+0000 7fe795ca5ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:48:28.574 INFO:teuthology.orchestra.run.smithi105.stdout:[client.1] 2024-09-21T09:48:28.575 INFO:teuthology.orchestra.run.smithi105.stdout: key = AQBslu5mx7z3IRAA5QEfEBMayaOSCTQ6pMK1xQ== 2024-09-21T09:48:28.623 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:28.366+0000 7fb5b7352ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:48:28.623 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:28.425+0000 7fb5b7352ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:48:28.623 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:28.485+0000 7fb5b7352ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:48:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-mon[31286]: from='client.? 172.21.15.105:0/1364664449' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:28 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:48:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-mon[39045]: from='client.? 172.21.15.105:0/1364664449' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:48:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-mon[27969]: from='client.? 172.21.15.105:0/1364664449' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-21T09:48:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:28 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-21T09:48:29.272 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:29.163+0000 7fe795ca5ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:48:29.273 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: [21/Sep/2024:09:48:29] ENGINE Bus STARTING 2024-09-21T09:48:29.273 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: CherryPy Checker: 2024-09-21T09:48:29.273 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: The Application mounted at '' has an empty config. 2024-09-21T09:48:29.273 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: 2024-09-21T09:48:29.273 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: [21/Sep/2024:09:48:29] ENGINE Serving on http://:::9283 2024-09-21T09:48:29.273 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: [21/Sep/2024:09:48:29] ENGINE Bus STARTED 2024-09-21T09:48:29.410 DEBUG:teuthology.orchestra.run.smithi105:> set -ex 2024-09-21T09:48:29.410 DEBUG:teuthology.orchestra.run.smithi105:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-21T09:48:29.411 DEBUG:teuthology.orchestra.run.smithi105:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-21T09:48:29.449 INFO:tasks.cephadm:Creating RBD pool 2024-09-21T09:48:29.449 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create rbd 8 2024-09-21T09:48:29.570 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.265+0000 7fb5b7352ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:29] ENGINE Bus STARTING 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: CherryPy Checker: 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: The Application mounted at '' has an empty config. 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.372+0000 7fb569533640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.372+0000 7fb569533640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.372+0000 7fb569533640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.372+0000 7fb569533640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.372+0000 7fb569533640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.373+0000 7fb56b537640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.571 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.373+0000 7fb56b537640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.572 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.373+0000 7fb56b537640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.572 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.373+0000 7fb56b537640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.572 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:29.373+0000 7fb56b537640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:48:29.572 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:29] ENGINE Serving on http://:::9283 2024-09-21T09:48:29.572 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:29] ENGINE Bus STARTED 2024-09-21T09:48:29.748 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:29.871 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: Standby manager daemon x restarted 2024-09-21T09:48:29.871 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: Standby manager daemon x started 2024-09-21T09:48:29.871 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: Active manager daemon y restarted 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: Activating manager daemon y 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: mgrmap e17: y(active, starting, since 0.00995891s), standbys: x 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:48:29.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: Manager daemon y is now available 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:29.873 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: Standby manager daemon x restarted 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: Standby manager daemon x started 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:48:29.874 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: Active manager daemon y restarted 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: Activating manager daemon y 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: mgrmap e17: y(active, starting, since 0.00995891s), standbys: x 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:48:29.875 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: Manager daemon y is now available 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:48:29.876 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: Standby manager daemon x restarted 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: Standby manager daemon x started 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/3099370946' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: Active manager daemon y restarted 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: Activating manager daemon y 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: osdmap e60: 8 total, 8 up, 8 in 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: mgrmap e17: y(active, starting, since 0.00995891s), standbys: x 2024-09-21T09:48:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:48:30.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:48:30.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:48:30.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:48:30.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: Manager daemon y is now available 2024-09-21T09:48:30.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:48:30.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:30.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:30.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:48:30.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:48:31.141 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:30.920+0000 7fb550101640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:48:30] ENGINE Error in HTTPServer.serve 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: Traceback (most recent call last): 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._connections.run(self.expiration_interval) 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._run(expiration_interval) 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: s = self.context.wrap_socket( 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: return self.sslsocket_class._create( 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:48:31.142 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self.do_handshake() 2024-09-21T09:48:31.143 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:48:31.143 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: self._sslobj.do_handshake() 2024-09-21T09:48:31.143 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:48:31.143 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:48:31.407 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:31 smithi033 ceph-mon[27969]: mgrmap e18: y(active, since 1.01168s), standbys: x 2024-09-21T09:48:31.407 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:31 smithi033 ceph-mon[27969]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:31.407 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:31 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:31.407 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:31 smithi033 ceph-mon[39045]: mgrmap e18: y(active, since 1.01168s), standbys: x 2024-09-21T09:48:31.407 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:31 smithi033 ceph-mon[39045]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:31.407 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:31 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:31.552 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:31 smithi105 ceph-mon[31286]: mgrmap e18: y(active, since 1.01168s), standbys: x 2024-09-21T09:48:31.552 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:31 smithi105 ceph-mon[31286]: pgmap v3: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:31.552 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:31 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:32.299 INFO:teuthology.orchestra.run.smithi033.stderr:pool 'rbd' created 2024-09-21T09:48:32.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: [21/Sep/2024:09:48:30] ENGINE Bus STARTING 2024-09-21T09:48:32.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: [21/Sep/2024:09:48:30] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:48:32.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: [21/Sep/2024:09:48:30] ENGINE Bus STARTED 2024-09-21T09:48:32.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: [21/Sep/2024:09:48:30] ENGINE Error in HTTPServer.serve 2024-09-21T09:48:32.621 INFO:journalctl@ceph.mon.c.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:48:32.621 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:48:32.622 INFO:journalctl@ceph.mon.c.smithi033.stdout: self.do_handshake() 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1959029039' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: [21/Sep/2024:09:48:30] ENGINE Bus STARTING 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: [21/Sep/2024:09:48:30] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:48:32.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: [21/Sep/2024:09:48:30] ENGINE Bus STARTED 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: [21/Sep/2024:09:48:30] ENGINE Error in HTTPServer.serve 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:48:32.624 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout: self.do_handshake() 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1959029039' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:32.625 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:32 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:32.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:48:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:48:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: [21/Sep/2024:09:48:30] ENGINE Bus STARTING 2024-09-21T09:48:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: [21/Sep/2024:09:48:30] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:48:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: [21/Sep/2024:09:48:30] ENGINE Bus STARTED 2024-09-21T09:48:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: [21/Sep/2024:09:48:30] ENGINE Error in HTTPServer.serve 2024-09-21T09:48:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-21T09:48:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:48:32.782 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-21T09:48:32.782 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:48:32.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: pgmap v4: 97 pgs: 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:32.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1959029039' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:48:32.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-21T09:48:32.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: mgrmap e19: y(active, since 2s), standbys: x 2024-09-21T09:48:32.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:32.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:32 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:32.944 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- rbd pool init rbd 2024-09-21T09:48:33.208 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.567 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.568 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.569 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.569 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-21T09:48:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: osdmap e61: 8 total, 8 up, 8 in 2024-09-21T09:48:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.300 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi105 to 2992M 2024-09-21T09:48:34.300 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: pgmap v6: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:34.301 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:48:34.301 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.301 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.301 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.301 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.301 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.302 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.302 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.302 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:34.302 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:34.302 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:34.302 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[27969]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:48:34.579 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi105 to 2992M 2024-09-21T09:48:34.579 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: pgmap v6: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:34.579 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:48:34.579 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.579 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:34.580 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:34 smithi033 ceph-mon[39045]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi105 to 2992M 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: pgmap v6: 105 pgs: 8 unknown, 97 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: osdmap e62: 8 total, 8 up, 8 in 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:48:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:34.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:48:34.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:48:34.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:48:34.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:34 smithi105 ceph-mon[31286]: mgrmap e20: y(active, since 4s), standbys: x 2024-09-21T09:48:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi033 to 2993M 2024-09-21T09:48:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:48:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:48:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:48:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:48:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:48:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3776083923' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:48:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:48:35.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi033 to 2993M 2024-09-21T09:48:35.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:48:35.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:48:35.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:48:35.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:48:35.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:48:35.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:48:35.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3776083923' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:48:35.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:35 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:48:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi033 to 2993M 2024-09-21T09:48:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:48:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:48:35.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:48:35.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:48:35.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: osdmap e63: 8 total, 8 up, 8 in 2024-09-21T09:48:35.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:48:35.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:48:35.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:48:35.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:48:35.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:35.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3776083923' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:48:35.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:35 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-21T09:48:36.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:36 smithi033 ceph-mon[27969]: Deploying daemon alertmanager.a on smithi033 2024-09-21T09:48:36.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:36 smithi033 ceph-mon[27969]: pgmap v9: 105 pgs: 5 unknown, 100 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T09:48:36.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:36 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:48:36.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:36 smithi033 ceph-mon[27969]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:48:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:36 smithi033 ceph-mon[39045]: Deploying daemon alertmanager.a on smithi033 2024-09-21T09:48:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:36 smithi033 ceph-mon[39045]: pgmap v9: 105 pgs: 5 unknown, 100 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T09:48:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:36 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:48:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:36 smithi033 ceph-mon[39045]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:48:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:36 smithi105 ceph-mon[31286]: Deploying daemon alertmanager.a on smithi033 2024-09-21T09:48:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:36 smithi105 ceph-mon[31286]: pgmap v9: 105 pgs: 5 unknown, 100 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T09:48:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:36 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-21T09:48:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:36 smithi105 ceph-mon[31286]: osdmap e64: 8 total, 8 up, 8 in 2024-09-21T09:48:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:37 smithi033 ceph-mon[39045]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:48:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:37 smithi033 ceph-mon[39045]: pgmap v12: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T09:48:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:37 smithi033 ceph-mon[27969]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:48:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:37 smithi033 ceph-mon[27969]: pgmap v12: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T09:48:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:37 smithi105 ceph-mon[31286]: osdmap e65: 8 total, 8 up, 8 in 2024-09-21T09:48:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:37 smithi105 ceph-mon[31286]: pgmap v12: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 0 B/s wr, 17 op/s 2024-09-21T09:48:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:38 smithi033 ceph-mon[39045]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:38 smithi033 ceph-mon[39045]: Cluster is now healthy 2024-09-21T09:48:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:38 smithi033 ceph-mon[39045]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:48:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:38 smithi033 ceph-mon[27969]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:38 smithi033 ceph-mon[27969]: Cluster is now healthy 2024-09-21T09:48:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:38 smithi033 ceph-mon[27969]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:48:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:38 smithi105 ceph-mon[31286]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:48:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:38 smithi105 ceph-mon[31286]: Cluster is now healthy 2024-09-21T09:48:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:38 smithi105 ceph-mon[31286]: osdmap e66: 8 total, 8 up, 8 in 2024-09-21T09:48:39.384 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph config log 1 --format=json 2024-09-21T09:48:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:39 smithi033 ceph-mon[39045]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T09:48:40.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:39 smithi033 ceph-mon[27969]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T09:48:40.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:39 smithi105 ceph-mon[31286]: pgmap v14: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 0 B/s wr, 14 op/s 2024-09-21T09:48:40.812 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:42.477 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:42 smithi033 ceph-mon[39045]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-21T09:48:42.477 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:42 smithi033 ceph-mon[39045]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:48:42.477 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:42 smithi033 ceph-mon[27969]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-21T09:48:42.478 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:42 smithi033 ceph-mon[27969]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:48:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:42 smithi105 ceph-mon[31286]: pgmap v15: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-21T09:48:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:42 smithi105 ceph-mon[31286]: osdmap e67: 8 total, 8 up, 8 in 2024-09-21T09:48:42.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:48:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:48:43.727 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:43 smithi033 podman[74407]: 2024-09-21 09:48:43.649983169 +0000 UTC m=+0.188659396 volume create 4f8fb96fa6ffd3735194341323fb31eee28a39eff30621dc2b220ea441ec505a 2024-09-21T09:48:43.854 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:48:43.977 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:43 smithi033 podman[74407]: 2024-09-21 09:48:43.834351068 +0000 UTC m=+0.373027294 container create a9a313d546e48b8d17e21b93e5903b5ef80384060a485dac7904ac47f1d17263 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:48:44.619 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-mon[39045]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-09-21T09:48:44.619 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1017691358' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:48:44.619 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:44.619 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-mon[27969]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-09-21T09:48:44.619 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1017691358' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:48:44.619 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:44.643 INFO:teuthology.orchestra.run.smithi033.stdout:[{"version":20,"timestamp":"2024-09-21T09:48:33.921313+0000","name":"","changes":[{"name":"osd/host:smithi033/osd_memory_target","previous_value":"4212219596","new_value":"3138477772"}]}] 2024-09-21T09:48:44.643 INFO:tasks.ceph_manager:config epoch is 20 2024-09-21T09:48:44.644 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-21T09:48:44.644 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-21T09:48:44.644 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph mgr dump --format=json 2024-09-21T09:48:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:44 smithi105 ceph-mon[31286]: pgmap v17: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 318 B/s wr, 0 op/s 2024-09-21T09:48:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:44 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1017691358' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-21T09:48:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:44 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:44.887 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 podman[74407]: 2024-09-21 09:48:44.777830793 +0000 UTC m=+1.316507041 container init a9a313d546e48b8d17e21b93e5903b5ef80384060a485dac7904ac47f1d17263 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 podman[74407]: 2024-09-21 09:48:44.783660865 +0000 UTC m=+1.322337091 container start a9a313d546e48b8d17e21b93e5903b5ef80384060a485dac7904ac47f1d17263 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:44.798Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:44.798Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:44.800Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.33 port=9094 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:44.801Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:44.829Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:44.829Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:44.833Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:44.833Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:48:44.888 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 bash[74407]: a9a313d546e48b8d17e21b93e5903b5ef80384060a485dac7904ac47f1d17263 2024-09-21T09:48:44.980 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:45.146 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:44 smithi033 systemd[1]: Started Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:48:45.146 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:45] ENGINE Bus STOPPING 2024-09-21T09:48:45.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:48:45.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:45] ENGINE Bus STOPPED 2024-09-21T09:48:45.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:45] ENGINE Bus STARTING 2024-09-21T09:48:45.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:45] ENGINE Serving on http://:::9283 2024-09-21T09:48:45.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:48:45] ENGINE Bus STARTED 2024-09-21T09:48:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: Deploying daemon grafana.a on smithi105 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[39045]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:48:46.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:48:46.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: Deploying daemon grafana.a on smithi105 2024-09-21T09:48:46.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:45 smithi033 ceph-mon[27969]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:48:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:48:46.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-21T09:48:46.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:46.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: Deploying daemon grafana.a on smithi105 2024-09-21T09:48:46.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:45 smithi105 ceph-mon[31286]: pgmap v18: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:48:46.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:46.802Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.0006157s 2024-09-21T09:48:47.170 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:48:47.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:47 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3799135061' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:48:47.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:47 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3799135061' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:48:47.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:47 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3799135061' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-21T09:48:47.893 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":20,"flags":0,"active_gid":14550,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6800","nonce":289485480},{"type":"v1","addr":"172.21.15.33:6801","nonce":289485480}]},"active_addr":"172.21.15.33:6801/289485480","active_change":"2024-09-21T09:48:29.268522+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24475,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.33:8443/","prometheus":"http://172.21.15.33:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3579553953}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":395059673}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":2390585401}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":131382457}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3299387945}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.33:0","nonce":3311171347}]}]} 2024-09-21T09:48:47.897 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-21T09:48:47.897 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-21T09:48:47.897 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:48:48.156 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:48.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:48 smithi033 ceph-mon[39045]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 217 B/s wr, 0 op/s 2024-09-21T09:48:48.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:48 smithi033 ceph-mon[27969]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 217 B/s wr, 0 op/s 2024-09-21T09:48:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:48 smithi105 ceph-mon[31286]: pgmap v19: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 217 B/s wr, 0 op/s 2024-09-21T09:48:50.045 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:48:50.045 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":67,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T09:48:41.423209+0000","last_up_change":"2024-09-21T09:46:55.803410+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:47:19.790061+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:47:22.976967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:48:32.209917+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6803","nonce":3277300496}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6805","nonce":3277300496}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6809","nonce":3277300496}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6807","nonce":3277300496}]},"public_addr":"172.21.15.33:6803/3277300496","cluster_addr":"172.21.15.33:6805/3277300496","heartbeat_back_addr":"172.21.15.33:6809/3277300496","heartbeat_front_addr":"172.21.15.33:6807/3277300496","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6811","nonce":2439650417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6813","nonce":2439650417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6817","nonce":2439650417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6815","nonce":2439650417}]},"public_addr":"172.21.15.33:6811/2439650417","cluster_addr":"172.21.15.33:6813/2439650417","heartbeat_back_addr":"172.21.15.33:6817/2439650417","heartbeat_front_addr":"172.21.15.33:6815/2439650417","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6819","nonce":3305661464}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6821","nonce":3305661464}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6825","nonce":3305661464}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6823","nonce":3305661464}]},"public_addr":"172.21.15.33:6819/3305661464","cluster_addr":"172.21.15.33:6821/3305661464","heartbeat_back_addr":"172.21.15.33:6825/3305661464","heartbeat_front_addr":"172.21.15.33:6823/3305661464","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6827","nonce":510343171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6829","nonce":510343171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6833","nonce":510343171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6831","nonce":510343171}]},"public_addr":"172.21.15.33:6827/510343171","cluster_addr":"172.21.15.33:6829/510343171","heartbeat_back_addr":"172.21.15.33:6833/510343171","heartbeat_front_addr":"172.21.15.33:6831/510343171","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6801","nonce":1234622360}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6803","nonce":1234622360}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6807","nonce":1234622360}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6805","nonce":1234622360}]},"public_addr":"172.21.15.105:6801/1234622360","cluster_addr":"172.21.15.105:6803/1234622360","heartbeat_back_addr":"172.21.15.105:6807/1234622360","heartbeat_front_addr":"172.21.15.105:6805/1234622360","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6809","nonce":1890006054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6811","nonce":1890006054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6815","nonce":1890006054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6813","nonce":1890006054}]},"public_addr":"172.21.15.105:6809/1890006054","cluster_addr":"172.21.15.105:6811/1890006054","heartbeat_back_addr":"172.21.15.105:6815/1890006054","heartbeat_front_addr":"172.21.15.105:6813/1890006054","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6817","nonce":592734975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6819","nonce":592734975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6823","nonce":592734975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6821","nonce":592734975}]},"public_addr":"172.21.15.105:6817/592734975","cluster_addr":"172.21.15.105:6819/592734975","heartbeat_back_addr":"172.21.15.105:6823/592734975","heartbeat_front_addr":"172.21.15.105:6821/592734975","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6825","nonce":455831701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6827","nonce":455831701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6831","nonce":455831701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6829","nonce":455831701}]},"public_addr":"172.21.15.105:6825/455831701","cluster_addr":"172.21.15.105:6827/455831701","heartbeat_back_addr":"172.21.15.105:6831/455831701","heartbeat_front_addr":"172.21.15.105:6829/455831701","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1a","mappings":[{"from":7,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.2f","mappings":[{"from":4,"to":3}]},{"pgid":"3.34","mappings":[{"from":4,"to":2}]},{"pgid":"3.39","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/150268354":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2645856759":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/3347076479":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/1156071037":"2024-09-22T09:48:29.268285+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6801/244283938":"2024-09-22T09:48:29.268285+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/244283938":"2024-09-22T09:48:29.268285+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:48:50.682 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:50 smithi033 ceph-mon[39045]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:48:50.682 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:50 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:50.682 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:50 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/447465761' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:50.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:50 smithi033 ceph-mon[27969]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:48:50.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:50 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:50.683 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:50 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/447465761' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:50.710 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-21T09:48:50.710 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:48:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:50 smithi105 ceph-mon[31286]: pgmap v20: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-21T09:48:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:50 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:50 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/447465761' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:50.967 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:52.448 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:52 smithi033 ceph-mon[39045]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:52.448 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:52 smithi033 ceph-mon[27969]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:52 smithi105 ceph-mon[31286]: pgmap v21: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:52.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:48:52 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:48:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:48:53.119 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:48:53.119 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":67,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T09:48:41.423209+0000","last_up_change":"2024-09-21T09:46:55.803410+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:47:19.790061+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:47:22.976967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:48:32.209917+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6803","nonce":3277300496}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6805","nonce":3277300496}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6809","nonce":3277300496}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6807","nonce":3277300496}]},"public_addr":"172.21.15.33:6803/3277300496","cluster_addr":"172.21.15.33:6805/3277300496","heartbeat_back_addr":"172.21.15.33:6809/3277300496","heartbeat_front_addr":"172.21.15.33:6807/3277300496","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6811","nonce":2439650417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6813","nonce":2439650417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6817","nonce":2439650417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6815","nonce":2439650417}]},"public_addr":"172.21.15.33:6811/2439650417","cluster_addr":"172.21.15.33:6813/2439650417","heartbeat_back_addr":"172.21.15.33:6817/2439650417","heartbeat_front_addr":"172.21.15.33:6815/2439650417","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6819","nonce":3305661464}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6821","nonce":3305661464}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6825","nonce":3305661464}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6823","nonce":3305661464}]},"public_addr":"172.21.15.33:6819/3305661464","cluster_addr":"172.21.15.33:6821/3305661464","heartbeat_back_addr":"172.21.15.33:6825/3305661464","heartbeat_front_addr":"172.21.15.33:6823/3305661464","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6827","nonce":510343171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6829","nonce":510343171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6833","nonce":510343171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6831","nonce":510343171}]},"public_addr":"172.21.15.33:6827/510343171","cluster_addr":"172.21.15.33:6829/510343171","heartbeat_back_addr":"172.21.15.33:6833/510343171","heartbeat_front_addr":"172.21.15.33:6831/510343171","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6801","nonce":1234622360}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6803","nonce":1234622360}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6807","nonce":1234622360}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6805","nonce":1234622360}]},"public_addr":"172.21.15.105:6801/1234622360","cluster_addr":"172.21.15.105:6803/1234622360","heartbeat_back_addr":"172.21.15.105:6807/1234622360","heartbeat_front_addr":"172.21.15.105:6805/1234622360","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6809","nonce":1890006054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6811","nonce":1890006054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6815","nonce":1890006054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6813","nonce":1890006054}]},"public_addr":"172.21.15.105:6809/1890006054","cluster_addr":"172.21.15.105:6811/1890006054","heartbeat_back_addr":"172.21.15.105:6815/1890006054","heartbeat_front_addr":"172.21.15.105:6813/1890006054","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6817","nonce":592734975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6819","nonce":592734975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6823","nonce":592734975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6821","nonce":592734975}]},"public_addr":"172.21.15.105:6817/592734975","cluster_addr":"172.21.15.105:6819/592734975","heartbeat_back_addr":"172.21.15.105:6823/592734975","heartbeat_front_addr":"172.21.15.105:6821/592734975","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6825","nonce":455831701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6827","nonce":455831701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6831","nonce":455831701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6829","nonce":455831701}]},"public_addr":"172.21.15.105:6825/455831701","cluster_addr":"172.21.15.105:6827/455831701","heartbeat_back_addr":"172.21.15.105:6831/455831701","heartbeat_front_addr":"172.21.15.105:6829/455831701","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1a","mappings":[{"from":7,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.2f","mappings":[{"from":4,"to":3}]},{"pgid":"3.34","mappings":[{"from":4,"to":2}]},{"pgid":"3.39","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/150268354":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2645856759":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/3347076479":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/1156071037":"2024-09-22T09:48:29.268285+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6801/244283938":"2024-09-22T09:48:29.268285+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/244283938":"2024-09-22T09:48:29.268285+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:48:53.431 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:53 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2938892587' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:53.431 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:53 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2938892587' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:53.726 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.0 flush_pg_stats 2024-09-21T09:48:53.726 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.1 flush_pg_stats 2024-09-21T09:48:53.727 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.2 flush_pg_stats 2024-09-21T09:48:53.727 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.3 flush_pg_stats 2024-09-21T09:48:53.727 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.4 flush_pg_stats 2024-09-21T09:48:53.728 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.5 flush_pg_stats 2024-09-21T09:48:53.728 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.6 flush_pg_stats 2024-09-21T09:48:53.729 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell osd.7 flush_pg_stats 2024-09-21T09:48:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:53 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2938892587' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:48:54.247 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:54.248 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:54.248 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:54.249 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:54.250 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:54.257 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:54.257 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:54.261 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:48:54.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:54 smithi105 ceph-mon[31286]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:54.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:54 smithi033 ceph-mon[27969]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:54.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:54 smithi033 ceph-mon[39045]: pgmap v22: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:54.859 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:48:54 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:48:54.804Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00284753s 2024-09-21T09:48:55.699 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 podman[63417]: 2024-09-21 09:48:55.566514264 +0000 UTC m=+0.113946959 container create 39f7f99588560a6b5c975934837c4f7b194df7ce1c4d30caf07965d21b442d58 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, vcs-type=git, architecture=x86_64) 2024-09-21T09:48:56.054 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 podman[63417]: 2024-09-21 09:48:55.802142052 +0000 UTC m=+0.349574754 container init 39f7f99588560a6b5c975934837c4f7b194df7ce1c4d30caf07965d21b442d58 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , version=8.5, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, architecture=x86_64, name=ubi8, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container) 2024-09-21T09:48:56.054 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 podman[63417]: 2024-09-21 09:48:55.806788631 +0000 UTC m=+0.354221327 container start 39f7f99588560a6b5c975934837c4f7b194df7ce1c4d30caf07965d21b442d58 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.buildah.version=1.32.0, vcs-type=git, name=ubi8, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, io.openshift.expose-services=, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, distribution-scope=public) 2024-09-21T09:48:56.054 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 bash[63417]: 39f7f99588560a6b5c975934837c4f7b194df7ce1c4d30caf07965d21b442d58 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 systemd[1]: Started Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.962707629Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.962926437Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.962956506Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.962968515Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.962980587Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.96299166Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.963002482Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.963014378Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.963026559Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.963037459Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.963047318Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:48:56.055 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.963057996Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.963068844Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=settings t=2024-09-21T09:48:55.963080004Z level=info msg="App mode production" 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=sqlstore t=2024-09-21T09:48:55.963144774Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=sqlstore t=2024-09-21T09:48:55.963172274Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.964300281Z level=info msg="Starting DB migrations" 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.964394511Z level=info msg="Executing migration" id="create migration_log table" 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.964938377Z level=info msg="Executing migration" id="create user table" 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.965456953Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.966075705Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.96652187Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-21T09:48:56.056 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.966930012Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.967312814Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.967975822Z level=info msg="Executing migration" id="create user table v2" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.968359772Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.968708366Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.969128078Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.969494965Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.96985331Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.970305936Z level=info msg="Executing migration" id="Update user table charset" 2024-09-21T09:48:56.057 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.970596804Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.971043156Z level=info msg="Executing migration" id="Add missing user data" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.971411395Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.97182073Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.972250632Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.97265897Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.973806195Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.974291596Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.974719703Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.97512045Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.978158533Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.980981468Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.98389749Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.984328007Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-21T09:48:56.058 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.987168287Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.987700781Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.990047787Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.992921029Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.993351554Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.996166542Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.99662124Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:55.999054007Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.001948139Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.002320572Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.005086135Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.005583269Z level=info msg="Executing migration" id="create star table" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.007754284Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.008228242Z level=info msg="Executing migration" id="create org table v1" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.010732296Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-21T09:48:56.059 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.013510313Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.016176973Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.01892736Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.021576561Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.022048144Z level=info msg="Executing migration" id="Update org table charset" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.024898111Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.027695689Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.030411822Z level=info msg="Executing migration" id="create dashboard table" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.032908854Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.035807933Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.038457979Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.041095284Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.042060078Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.042729025Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-21T09:48:56.060 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.043622707Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.044416092Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.045441228Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.046151458Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.046989946Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.048001479Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.048682176Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.049692777Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.050413988Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.051403651Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.052114405Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.053004337Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.053828411Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.05466606Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-21T09:48:56.061 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.05547382Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-21T09:48:56.306 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.055790735Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-21T09:48:56.306 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.056303248Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-21T09:48:56.306 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.056800015Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-21T09:48:56.306 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.057649242Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-21T09:48:56.306 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.058535403Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-21T09:48:56.306 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.059411037Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-21T09:48:56.306 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.060308803Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.061130825Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.062125083Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.06289046Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.063894446Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.064843597Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.065795584Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.06658062Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.067413597Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.068259191Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.069091701Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-21T09:48:56.307 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.069992965Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-21T09:48:56.308 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.070839129Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-21T09:48:56.308 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.073594019Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-21T09:48:56.308 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.076466015Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-21T09:48:56.308 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.07923704Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.081924564Z level=info msg="Executing migration" id="create data_source table" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.084634478Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.08731576Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.08999326Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.092904135Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.095405871Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.098439221Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.101080729Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.103884425Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.106566742Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.109212721Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.111992195Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.114682575Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-21T09:48:56.309 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.117568608Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.120343811Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.123027117Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.125781224Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.128381136Z level=info msg="Executing migration" id="Add uid column" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.131143951Z level=info msg="Executing migration" id="Update uid value" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.133956354Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.136621874Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.139253988Z level=info msg="Executing migration" id="create api_key table" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.14196702Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.144703356Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.147340398Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.150151375Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.152592585Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-21T09:48:56.310 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.155285869Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-21T09:48:56.311 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.155709228Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-21T09:48:56.311 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.158402539Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-21T09:48:56.311 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.161121295Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-21T09:48:56.311 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.163939405Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-21T09:48:56.313 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.164519937Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-21T09:48:56.313 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.165437685Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-21T09:48:56.313 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.16627662Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-21T09:48:56.313 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.167142467Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-21T09:48:56.313 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.16746904Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.16829588Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.168852261Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.169178746Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.169940498Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.170494771Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.170885035Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.171260518Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.171683236Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.172118539Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.172586781Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.173452239Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.173777468Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.174767824Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-21T09:48:56.314 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.175486524Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.176505991Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.176855164Z level=info msg="Executing migration" id="create quota table v1" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.17776526Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.178814267Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.17959305Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.180189811Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.181211426Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.181966508Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.182993732Z level=info msg="Executing migration" id="create session table" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.183490628Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.184278972Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.184614716Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.185164178Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.186189165Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-21T09:48:56.315 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.186520218Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.187462361Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.18801249Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.19029065Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.193145714Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.19354208Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.196309004Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.199109793Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.201939098Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.20270266Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.205413542Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.208199539Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.211086744Z level=info msg="Executing migration" id="create alert table v1" 2024-09-21T09:48:56.316 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.211565799Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.214363056Z level=info msg="Executing migration" id="add index alert state" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.217036442Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.219819069Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.220208483Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.223160548Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.225616792Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.228226087Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.231146147Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.23360135Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.236393715Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.238921676Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.239322084Z level=info msg="Executing migration" id="Add column is_default" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.241961371Z level=info msg="Executing migration" id="Add column frequency" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.242628783Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-21T09:48:56.317 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.24479809Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.247613403Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.248195317Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.250897894Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.253473934Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.256162225Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.256589606Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.259385118Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.259819699Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.262127621Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.262822456Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.263507657Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.264277963Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.264739632Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-21T09:48:56.318 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.265158593Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-21T09:48:56.319 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.26596062Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-21T09:48:56.319 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.266280425Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-21T09:48:56.319 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.267153596Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-21T09:48:56.319 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.267608759Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-21T09:48:56.319 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.268033161Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-21T09:48:56.319 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.2685119Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-21T09:48:56.319 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.268978372Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-21T09:48:56.319 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.269405061Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.270265408Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.271060941Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.271866167Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.272188185Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.273013209Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.273439458Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.274105767Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.274955067Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.275399948Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.276204824Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.277787125Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.278223128Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.279041045Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-21T09:48:56.320 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.279389495Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.280195954Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.280536166Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.281217678Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.282074163Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.282522721Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.283397531Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.284216042Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.285231983Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.285811818Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.286783184Z level=info msg="Executing migration" id="Move region to single row" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.289601194Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.292370819Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.292805703Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-21T09:48:56.321 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.295575157Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-21T09:48:56.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.298548148Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-21T09:48:56.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.301281188Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-21T09:48:56.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.301761904Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-21T09:48:56.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.304647107Z level=info msg="Executing migration" id="create test_data table" 2024-09-21T09:48:56.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.307273937Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-21T09:48:56.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.307702275Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-21T09:48:56.322 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.308162395Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[39045]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[27969]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:56 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-mon[31286]: pgmap v23: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:56.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.311003027Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.313889772Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.316650312Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.316983649Z level=info msg="Executing migration" id="create team table" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.31997757Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.322690263Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.325405005Z level=info msg="Executing migration" id="create team member table" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.325999797Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.328862075Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.33151693Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-21T09:48:56.558 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.334260144Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.33498734Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.337924855Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.340539272Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.343375857Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.34612711Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.346633568Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.347097037Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.347574058Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.350307867Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.350739718Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.351155306Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.351619187Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.351902786Z level=info msg="Executing migration" id="create tag table" 2024-09-21T09:48:56.559 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.352297558Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.352734982Z level=info msg="Executing migration" id="create login attempt table" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.353171232Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.353601437Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.354041626Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.355928365Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.356330631Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.356778383Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.35711827Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.357497065Z level=info msg="Executing migration" id="create user auth table" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.357850218Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.35830163Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.358616628Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-21T09:48:56.560 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.359388988Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.360167451Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.360933825Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.361726015Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.362191951Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.36292884Z level=info msg="Executing migration" id="create server_lock table" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.363335224Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.363770944Z level=info msg="Executing migration" id="create user auth token table" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.364211578Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.364638819Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.365108552Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.365541271Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.36631957Z level=info msg="Executing migration" id="create cache_data table" 2024-09-21T09:48:56.562 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.366728266Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.367187541Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.367625949Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.370328424Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.370655307Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.370964252Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.371486149Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.374340618Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.377012133Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.377361098Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.379561455Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.380047757Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.382358853Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.383235977Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.384128483Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-21T09:48:56.563 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.384598823Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.385463722Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.385967959Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.386486789Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.387379962Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.387797928Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.388321142Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.389124953Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.389630744Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.390109157Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.390971524Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.391452974Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.392249716Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.395667981Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-21T09:48:56.564 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.399048991Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.399912862Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.400382385Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.401272904Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.401724377Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.40256387Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.403524888Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.404232387Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.405216936Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.406118192Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.406989689Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.407886779Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.40875908Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.40960375Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.41047267Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-21T09:48:56.565 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.411326678Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.412182908Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.413055445Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.413893001Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.416739825Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.419680771Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.422420516Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.422919845Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.423269615Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.426138207Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.428909304Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.429790482Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.43066889Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.431559066Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.432408697Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-21T09:48:56.566 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.432837508Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.433711674Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.434552479Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.435460081Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.435949323Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.43680275Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.437638052Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.438485574Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.439392616Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.440272304Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.441119415Z level=info msg="Executing migration" id="create alert_image table" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.441960619Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.442417938Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.443236432Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-21T09:48:56.567 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.443667046Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.444179465Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.444292736Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.445027342Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.445425135Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.448101383Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.448244799Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.448382426Z level=info msg="alerts found to migrate" alerts=0 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.448827258Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.450943549Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.453857342Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.45449603Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.457202788Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.460186352Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.463021658Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-21T09:48:56.568 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.463461676Z level=info msg="Executing migration" id="create data_keys table" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.46618169Z level=info msg="Executing migration" id="create secrets table" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.46898346Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.473428754Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.47432297Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.477154834Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.481540501Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.485914856Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.486326474Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.488949404Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.491787175Z level=info msg="Executing migration" id="create permission table" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.494302699Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.494763055Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.497369615Z level=info msg="Executing migration" id="create role table" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.500008852Z level=info msg="Executing migration" id="add column display_name" 2024-09-21T09:48:56.569 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.502648287Z level=info msg="Executing migration" id="add column group_name" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.503555153Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.506240689Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.508986848Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.511708665Z level=info msg="Executing migration" id="create team role table" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.512121646Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.514790188Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.517622033Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.520286124Z level=info msg="Executing migration" id="create user role table" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.520703652Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.523507904Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.524255229Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.525086502Z level=info msg="Executing migration" id="create builtin role table" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.525907263Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.526757096Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-21T09:48:56.570 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.527585819Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.528635297Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.529212013Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.530003048Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.53079081Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.531656916Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.532065371Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.532884093Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.533844444Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.534687928Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.535157858Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.535998815Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.536821128Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.537679575Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.538083483Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-21T09:48:56.571 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.538958162Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.53980822Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.540645758Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.541098028Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.542153956Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.54300741Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.543894213Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.544344494Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.545196288Z level=info msg="Executing migration" id="add correlation config column" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.546198685Z level=info msg="Executing migration" id="create entity_events table" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.547074015Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.547484435Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.547578314Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.548356558Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.548449712Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:48:56.572 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.54931983Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-21T09:48:56.573 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.550064667Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-21T09:48:56.573 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.550502631Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-21T09:48:56.573 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.553330999Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-21T09:48:56.573 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.556138775Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:48:56.573 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.558967813Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:48:57.030 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.561705053Z level=info msg="Executing migration" id="Drop public config table" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.564434356Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.567200385Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.570146201Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.570641811Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.573556543Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.576610461Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.579395572Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.580406739Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.583282412Z level=info msg="Executing migration" id="add share column" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.585945541Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-21T09:48:57.031 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.588673372Z level=info msg="Executing migration" id="create file table" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.589131115Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.591905722Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.594513504Z level=info msg="Executing migration" id="create file_meta table" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.597114939Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.59758819Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.600369453Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.60094231Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.603140541Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.605937035Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.607102282Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-21T09:48:57.032 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.609799212Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.612432682Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.615078841Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.615418696Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.618184604Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.619271274Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.620954393Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.623666179Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.633604133Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.63641796Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.637083336Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.640834647Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-21T09:48:57.033 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.641169961Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.641997367Z level=info msg="Executing migration" id="create folder table" 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.642427795Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.642864448Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.643736458Z level=info msg="Executing migration" id="Update folder title length" 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.644041288Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=migrator t=2024-09-21T09:48:56.646750785Z level=info msg="migrations completed" performed=485 skipped=0 duration=682.3799ms 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=sqlstore t=2024-09-21T09:48:56.647239635Z level=info msg="Created default organization" 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=plugin.loader t=2024-09-21T09:48:56.707794813Z level=info msg="Plugin registered" pluginID=input 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=plugin.loader t=2024-09-21T09:48:56.735915752Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=plugin.loader t=2024-09-21T09:48:56.735942032Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=secrets t=2024-09-21T09:48:56.735988041Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:48:57.034 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=query_data t=2024-09-21T09:48:56.737639093Z level=info msg="Query Service initialization" 2024-09-21T09:48:57.035 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=live.push_http t=2024-09-21T09:48:56.739996748Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:48:57.608 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-mon[31286]: Deploying daemon node-exporter.a on smithi033 2024-09-21T09:48:57.608 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=infra.usagestats.collector t=2024-09-21T09:48:57.464461844Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:48:57.608 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=provisioning.datasources t=2024-09-21T09:48:57.464776439Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-21T09:48:57.608 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=provisioning.datasources t=2024-09-21T09:48:57.498258876Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:48:57.608 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=provisioning.datasources t=2024-09-21T09:48:57.530046605Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=provisioning.alerting t=2024-09-21T09:48:57.551294736Z level=info msg="starting to provision alerting" 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=provisioning.alerting t=2024-09-21T09:48:57.551314351Z level=info msg="finished to provision alerting" 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=ngalert.state.manager t=2024-09-21T09:48:57.551400286Z level=info msg="Warming state cache for startup" 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=grafanaStorageLogger t=2024-09-21T09:48:57.55150459Z level=info msg="storage starting" 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=http.server t=2024-09-21T09:48:57.553436007Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=ngalert.state.manager t=2024-09-21T09:48:57.561722089Z level=info msg="State cache has been initialized" states=0 duration=10.318929ms 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:48:57.561857248Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=ticker t=2024-09-21T09:48:57.56194925Z level=info msg=starting first_tick=2024-09-21T09:49:00Z 2024-09-21T09:48:57.609 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=sqlstore.transactions t=2024-09-21T09:48:57.607135571Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T09:48:57.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:57 smithi033 ceph-mon[39045]: Deploying daemon node-exporter.a on smithi033 2024-09-21T09:48:57.714 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:57 smithi033 ceph-mon[27969]: Deploying daemon node-exporter.a on smithi033 2024-09-21T09:48:58.030 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:48:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[63431]: logger=sqlstore.transactions t=2024-09-21T09:48:57.617904688Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-09-21T09:48:58.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:58 smithi033 ceph-mon[39045]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:58.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:58 smithi033 ceph-mon[27969]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:58 smithi105 ceph-mon[31286]: pgmap v24: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:48:59.705 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:59 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:59.705 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:48:59 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:59.706 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:59 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:59.706 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:48:59 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:48:59.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:59 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:48:59.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:48:59 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:00.709 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:00 smithi033 ceph-mon[39045]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:00.709 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:00 smithi033 ceph-mon[27969]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:00 smithi105 ceph-mon[31286]: pgmap v25: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:02.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:02 smithi033 ceph-mon[27969]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:02.650 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:02 smithi033 ceph-mon[39045]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:02 smithi105 ceph-mon[31286]: pgmap v26: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:02.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:49:02 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:49:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:49:04.613 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:04 smithi033 ceph-mon[27969]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:04.613 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:04 smithi033 ceph-mon[39045]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:04 smithi105 ceph-mon[31286]: pgmap v27: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:05.586 INFO:teuthology.orchestra.run.smithi033.stdout:206158430236 2024-09-21T09:49:05.586 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.7 2024-09-21T09:49:06.724 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:06 smithi033 ceph-mon[39045]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:06.724 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:06 smithi033 ceph-mon[27969]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:06 smithi105 ceph-mon[31286]: pgmap v28: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:07.021 INFO:teuthology.orchestra.run.smithi033.stdout:34359738429 2024-09-21T09:49:07.021 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.0 2024-09-21T09:49:07.856 INFO:teuthology.orchestra.run.smithi033.stdout:55834574905 2024-09-21T09:49:07.856 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.1 2024-09-21T09:49:08.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:08 smithi033 ceph-mon[39045]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:08.677 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:08 smithi033 ceph-mon[27969]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:08.707 INFO:teuthology.orchestra.run.smithi033.stdout:77309411379 2024-09-21T09:49:08.707 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.2 2024-09-21T09:49:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:08 smithi105 ceph-mon[31286]: pgmap v29: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:09.333 INFO:teuthology.orchestra.run.smithi033.stdout:180388626464 2024-09-21T09:49:09.333 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.6 2024-09-21T09:49:10.246 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:10 smithi033 bash[76672]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:49:10.361 INFO:teuthology.orchestra.run.smithi033.stdout:128849018923 2024-09-21T09:49:10.362 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.4 2024-09-21T09:49:10.597 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:10 smithi033 bash[76672]: Getting image source signatures 2024-09-21T09:49:10.597 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:10 smithi033 bash[76672]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:49:10.597 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:10 smithi033 bash[76672]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:49:10.597 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:10 smithi033 bash[76672]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:49:10.597 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:10 smithi033 ceph-mon[39045]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:10.598 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:10 smithi033 ceph-mon[27969]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:10 smithi105 ceph-mon[31286]: pgmap v30: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:12.059 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:12.345 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:12 smithi033 bash[76672]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:49:12.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:12 smithi033 ceph-mon[27969]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:12 smithi033 ceph-mon[39045]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:12.746 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:12 smithi033 bash[76672]: Writing manifest to image destination 2024-09-21T09:49:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:12 smithi105 ceph-mon[31286]: pgmap v31: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:12.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:49:12 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:49:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:49:12.822 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:12.822 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:13.186 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:12 smithi033 podman[76672]: 2024-09-21 09:49:12.860583122 +0000 UTC m=+7.685966009 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:49:13.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 podman[76672]: 2024-09-21 09:49:13.184904285 +0000 UTC m=+8.010287160 container create ae258c372f477343af831e4b4e311019a262336dcf96bb41a2daa9537fefbc12 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:13.792 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 podman[76672]: 2024-09-21 09:49:13.522478789 +0000 UTC m=+8.347861668 container init ae258c372f477343af831e4b4e311019a262336dcf96bb41a2daa9537fefbc12 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:13.792 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 podman[76672]: 2024-09-21 09:49:13.528629548 +0000 UTC m=+8.354012413 container start ae258c372f477343af831e4b4e311019a262336dcf96bb41a2daa9537fefbc12 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:13.792 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.532Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.532Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.533Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.533Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.533Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.533Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:49:13.793 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:49:13.794 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:49:13.794 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:49:13.794 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:49:13.794 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:49:13.794 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:49:13.794 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:49:13.794 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:49:13.794 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:49:13.795 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:49:13.796 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:49:13.796 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:49:13.796 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:49:13.796 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:49:13.796 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:49:13.796 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:49:13.796 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:49:13.796 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[77480]: ts=2024-09-21T09:49:13.534Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 bash[76672]: ae258c372f477343af831e4b4e311019a262336dcf96bb41a2daa9537fefbc12 2024-09-21T09:49:13.797 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:49:13 smithi033 systemd[1]: Started Ceph node-exporter.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:14.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:14 smithi105 ceph-mon[31286]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:14.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:14 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:14 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:14 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:14 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[39045]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[27969]: pgmap v32: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.602 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:14.603 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:14 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:15.030 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:14 smithi105 bash[63709]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-21T09:49:15.312 INFO:teuthology.orchestra.run.smithi033.stdout:107374182449 2024-09-21T09:49:15.313 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.3 2024-09-21T09:49:15.381 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:15 smithi105 bash[63709]: Getting image source signatures 2024-09-21T09:49:15.381 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:15 smithi105 bash[63709]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-21T09:49:15.381 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:15 smithi105 bash[63709]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-21T09:49:15.381 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:15 smithi105 bash[63709]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-21T09:49:15.546 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:15 smithi033 ceph-mon[39045]: Deploying daemon node-exporter.b on smithi105 2024-09-21T09:49:15.546 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:15 smithi033 ceph-mon[27969]: Deploying daemon node-exporter.b on smithi105 2024-09-21T09:49:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:15 smithi105 ceph-mon[31286]: Deploying daemon node-exporter.b on smithi105 2024-09-21T09:49:16.276 INFO:teuthology.orchestra.run.smithi033.stdout:154618822695 2024-09-21T09:49:16.276 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd last-stat-seq osd.5 2024-09-21T09:49:16.306 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:16.312 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:16.313 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:16.640 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:16 smithi105 ceph-mon[31286]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:16.640 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:16 smithi105 bash[63709]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-21T09:49:16.643 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:16 smithi033 ceph-mon[39045]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:16.644 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:16 smithi033 ceph-mon[27969]: pgmap v33: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:16.990 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:16 smithi105 bash[63709]: Writing manifest to image destination 2024-09-21T09:49:17.280 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:16 smithi105 podman[63709]: 2024-09-21 09:49:16.903032373 +0000 UTC m=+2.234118378 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:16 smithi105 podman[63709]: 2024-09-21 09:49:16.992925135 +0000 UTC m=+2.324011182 container create 446328798f333717972f40f1413ed20a3dc9f9a994b803faf822b21c19d67d64 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 podman[63709]: 2024-09-21 09:49:17.211367398 +0000 UTC m=+2.542453422 container init 446328798f333717972f40f1413ed20a3dc9f9a994b803faf822b21c19d67d64 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 podman[63709]: 2024-09-21 09:49:17.216931048 +0000 UTC m=+2.548017053 container start 446328798f333717972f40f1413ed20a3dc9f9a994b803faf822b21c19d67d64 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.221Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.221Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.221Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.221Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.221Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.221Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:49:17.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:49:17.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:49:17.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:49:17.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:49:17.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:49:17.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:49:17.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:49:17.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:49:17.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:49:17.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:49:17.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:49:17.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:49:17.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:49:17.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:49:17.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:49:17.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:49:17.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.222Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:49:17.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.223Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:49:17.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[63782]: ts=2024-09-21T09:49:17.223Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:49:17.780 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 bash[63709]: 446328798f333717972f40f1413ed20a3dc9f9a994b803faf822b21c19d67d64 2024-09-21T09:49:17.780 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:49:17 smithi105 systemd[1]: Started Ceph node-exporter.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:18.213 INFO:teuthology.orchestra.run.smithi033.stdout:55834574907 2024-09-21T09:49:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2422856976' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2517754870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2422856976' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:49:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:18 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2517754870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:49:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: pgmap v34: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:18.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:18.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2422856976' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-21T09:49:18.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:18 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2517754870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-21T09:49:18.925 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574905 got 55834574907 for osd.1 2024-09-21T09:49:18.925 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:19.853 INFO:teuthology.orchestra.run.smithi033.stdout:206158430239 2024-09-21T09:49:20.037 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:20 smithi033 ceph-mon[39045]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:20 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:20.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:20 smithi033 ceph-mon[27969]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:20.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:20 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:20.530 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430236 got 206158430239 for osd.7 2024-09-21T09:49:20.531 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:20.676 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:20 smithi105 ceph-mon[31286]: pgmap v35: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:20.677 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:20 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:22.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:22 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:22.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:22 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:22.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:22 smithi033 ceph-mon[39045]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:22.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:22 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:22.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:22 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:22.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:22 smithi033 ceph-mon[27969]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:22.530 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:49:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:49:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:49:22.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:22 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:22.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:22 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:22.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:22 smithi105 ceph-mon[31286]: pgmap v36: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:22.911 INFO:teuthology.orchestra.run.smithi033.stdout:34359738433 2024-09-21T09:49:23.094 INFO:teuthology.orchestra.run.smithi033.stdout:180388626468 2024-09-21T09:49:23.209 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:23 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2655042019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:49:23.209 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:23 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1916616528' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:49:23.209 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:23 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2655042019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:49:23.209 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:23 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1916616528' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:49:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:23 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2655042019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-21T09:49:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:23 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1916616528' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-21T09:49:23.845 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626464 got 180388626468 for osd.6 2024-09-21T09:49:23.846 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:24.354 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:24 smithi033 ceph-mon[39045]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:24.354 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:24 smithi033 ceph-mon[27969]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:24 smithi105 ceph-mon[31286]: pgmap v37: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:25.883 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738429 got 34359738433 for osd.0 2024-09-21T09:49:25.883 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:26.444 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:26 smithi033 ceph-mon[39045]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:26.444 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:26 smithi033 ceph-mon[27969]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:26 smithi105 ceph-mon[31286]: pgmap v38: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:26.897 INFO:teuthology.orchestra.run.smithi033.stdout:128849018927 2024-09-21T09:49:27.074 INFO:teuthology.orchestra.run.smithi033.stdout:77309411384 2024-09-21T09:49:27.275 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:27.728 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:27 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/658160414' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:49:27.728 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:27 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2895473634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:49:27.728 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:27 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/658160414' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:49:27.728 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:27 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2895473634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:49:27.752 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411379 got 77309411384 for osd.2 2024-09-21T09:49:27.752 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:27 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/658160414' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-21T09:49:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:27 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2895473634' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-21T09:49:28.161 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018923 got 128849018927 for osd.4 2024-09-21T09:49:28.161 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:28.424 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:28 smithi033 ceph-mon[39045]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:28.424 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:28 smithi033 ceph-mon[27969]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:28.587 INFO:teuthology.orchestra.run.smithi033.stdout:107374182452 2024-09-21T09:49:28.681 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:28 smithi105 ceph-mon[31286]: pgmap v39: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:29.342 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182449 got 107374182452 for osd.3 2024-09-21T09:49:29.342 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:29.695 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:29 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1953198837' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:49:29.695 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:29 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1953198837' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:49:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:29.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:29 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1953198837' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-21T09:49:29.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:30.664 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:30 smithi033 ceph-mon[39045]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:30.665 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:30 smithi033 ceph-mon[27969]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:30.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:30 smithi105 ceph-mon[31286]: pgmap v40: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:31.688 INFO:teuthology.orchestra.run.smithi033.stdout:154618822699 2024-09-21T09:49:32.427 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822695 got 154618822699 for osd.5 2024-09-21T09:49:32.427 DEBUG:teuthology.parallel:result is None 2024-09-21T09:49:32.428 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-21T09:49:32.428 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:49:32.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:32 smithi033 ceph-mon[39045]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:32.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:32 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1323637795' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:49:32.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:32 smithi033 ceph-mon[27969]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:32.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:32 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1323637795' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:49:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:32 smithi105 ceph-mon[31286]: pgmap v41: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:32 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1323637795' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-21T09:49:32.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:49:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:49:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:49:33.322 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:34 smithi033 ceph-mon[39045]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:34.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:34 smithi033 ceph-mon[27969]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:34.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:34 smithi105 ceph-mon[31286]: pgmap v42: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:35.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:35.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:35.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:35.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:35.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:35 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:35.852 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:49:35.853 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-21T09:49:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:35 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:36.852 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:36 smithi033 ceph-mon[39045]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:36.852 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:36 smithi033 ceph-mon[39045]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:49:36.852 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:36 smithi033 ceph-mon[39045]: Reconfiguring daemon alertmanager.a on smithi033 2024-09-21T09:49:36.853 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:36 smithi033 ceph-mon[27969]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:36.853 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:36 smithi033 ceph-mon[27969]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:49:36.853 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:36 smithi033 ceph-mon[27969]: Reconfiguring daemon alertmanager.a on smithi033 2024-09-21T09:49:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:36 smithi105 ceph-mon[31286]: pgmap v43: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:36 smithi105 ceph-mon[31286]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:49:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:36 smithi105 ceph-mon[31286]: Reconfiguring daemon alertmanager.a on smithi033 2024-09-21T09:49:37.249 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":43,"stamp":"2024-09-21T09:49:35.298278+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386052,"kb_used_data":7204,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575164,"statfs":{"total":767960285184,"available":765516967936,"internally_reserved":0,"allocated":7376896,"data_stored":4256321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002597"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978641+0000","last_change":"2024-09-21T09:47:24.066961+0000","last_active":"2024-09-21T09:48:37.978641+0000","last_peered":"2024-09-21T09:48:37.978641+0000","last_clean":"2024-09-21T09:48:37.978641+0000","last_became_active":"2024-09-21T09:47:24.066672+0000","last_became_peered":"2024-09-21T09:47:24.066672+0000","last_unstale":"2024-09-21T09:48:37.978641+0000","last_undegraded":"2024-09-21T09:48:37.978641+0000","last_fullsized":"2024-09-21T09:48:37.978641+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861122+0000","last_change":"2024-09-21T09:47:24.059165+0000","last_active":"2024-09-21T09:48:37.861122+0000","last_peered":"2024-09-21T09:48:37.861122+0000","last_clean":"2024-09-21T09:48:37.861122+0000","last_became_active":"2024-09-21T09:47:24.058702+0000","last_became_peered":"2024-09-21T09:47:24.058702+0000","last_unstale":"2024-09-21T09:48:37.861122+0000","last_undegraded":"2024-09-21T09:48:37.861122+0000","last_fullsized":"2024-09-21T09:48:37.861122+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887263+0000","last_change":"2024-09-21T09:47:24.060359+0000","last_active":"2024-09-21T09:48:37.887263+0000","last_peered":"2024-09-21T09:48:37.887263+0000","last_clean":"2024-09-21T09:48:37.887263+0000","last_became_active":"2024-09-21T09:47:24.060088+0000","last_became_peered":"2024-09-21T09:47:24.060088+0000","last_unstale":"2024-09-21T09:48:37.887263+0000","last_undegraded":"2024-09-21T09:48:37.887263+0000","last_fullsized":"2024-09-21T09:48:37.887263+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865595+0000","last_change":"2024-09-21T09:47:24.065047+0000","last_active":"2024-09-21T09:48:37.865595+0000","last_peered":"2024-09-21T09:48:37.865595+0000","last_clean":"2024-09-21T09:48:37.865595+0000","last_became_active":"2024-09-21T09:47:24.064743+0000","last_became_peered":"2024-09-21T09:47:24.064743+0000","last_unstale":"2024-09-21T09:48:37.865595+0000","last_undegraded":"2024-09-21T09:48:37.865595+0000","last_fullsized":"2024-09-21T09:48:37.865595+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978224+0000","last_change":"2024-09-21T09:47:24.069692+0000","last_active":"2024-09-21T09:48:37.978224+0000","last_peered":"2024-09-21T09:48:37.978224+0000","last_clean":"2024-09-21T09:48:37.978224+0000","last_became_active":"2024-09-21T09:47:24.069090+0000","last_became_peered":"2024-09-21T09:47:24.069090+0000","last_unstale":"2024-09-21T09:48:37.978224+0000","last_undegraded":"2024-09-21T09:48:37.978224+0000","last_fullsized":"2024-09-21T09:48:37.978224+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865499+0000","last_change":"2024-09-21T09:47:24.053540+0000","last_active":"2024-09-21T09:48:37.865499+0000","last_peered":"2024-09-21T09:48:37.865499+0000","last_clean":"2024-09-21T09:48:37.865499+0000","last_became_active":"2024-09-21T09:47:24.053163+0000","last_became_peered":"2024-09-21T09:47:24.053163+0000","last_unstale":"2024-09-21T09:48:37.865499+0000","last_undegraded":"2024-09-21T09:48:37.865499+0000","last_fullsized":"2024-09-21T09:48:37.865499+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978070+0000","last_change":"2024-09-21T09:47:46.774760+0000","last_active":"2024-09-21T09:48:37.978070+0000","last_peered":"2024-09-21T09:48:37.978070+0000","last_clean":"2024-09-21T09:48:37.978070+0000","last_became_active":"2024-09-21T09:47:46.774445+0000","last_became_peered":"2024-09-21T09:47:46.774445+0000","last_unstale":"2024-09-21T09:48:37.978070+0000","last_undegraded":"2024-09-21T09:48:37.978070+0000","last_fullsized":"2024-09-21T09:48:37.978070+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079815+0000","last_change":"2024-09-21T09:47:24.064091+0000","last_active":"2024-09-21T09:48:38.079815+0000","last_peered":"2024-09-21T09:48:38.079815+0000","last_clean":"2024-09-21T09:48:38.079815+0000","last_became_active":"2024-09-21T09:47:24.063698+0000","last_became_peered":"2024-09-21T09:47:24.063698+0000","last_unstale":"2024-09-21T09:48:38.079815+0000","last_undegraded":"2024-09-21T09:48:38.079815+0000","last_fullsized":"2024-09-21T09:48:38.079815+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978791+0000","last_change":"2024-09-21T09:47:24.071747+0000","last_active":"2024-09-21T09:48:37.978791+0000","last_peered":"2024-09-21T09:48:37.978791+0000","last_clean":"2024-09-21T09:48:37.978791+0000","last_became_active":"2024-09-21T09:47:24.071475+0000","last_became_peered":"2024-09-21T09:47:24.071475+0000","last_unstale":"2024-09-21T09:48:37.978791+0000","last_undegraded":"2024-09-21T09:48:37.978791+0000","last_fullsized":"2024-09-21T09:48:37.978791+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080913+0000","last_change":"2024-09-21T09:47:24.058216+0000","last_active":"2024-09-21T09:48:38.080913+0000","last_peered":"2024-09-21T09:48:38.080913+0000","last_clean":"2024-09-21T09:48:38.080913+0000","last_became_active":"2024-09-21T09:47:24.057940+0000","last_became_peered":"2024-09-21T09:47:24.057940+0000","last_unstale":"2024-09-21T09:48:38.080913+0000","last_undegraded":"2024-09-21T09:48:38.080913+0000","last_fullsized":"2024-09-21T09:48:38.080913+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080816+0000","last_change":"2024-09-21T09:47:24.053543+0000","last_active":"2024-09-21T09:48:38.080816+0000","last_peered":"2024-09-21T09:48:38.080816+0000","last_clean":"2024-09-21T09:48:38.080816+0000","last_became_active":"2024-09-21T09:47:24.053264+0000","last_became_peered":"2024-09-21T09:47:24.053264+0000","last_unstale":"2024-09-21T09:48:38.080816+0000","last_undegraded":"2024-09-21T09:48:38.080816+0000","last_fullsized":"2024-09-21T09:48:38.080816+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861618+0000","last_change":"2024-09-21T09:47:46.850968+0000","last_active":"2024-09-21T09:48:37.861618+0000","last_peered":"2024-09-21T09:48:37.861618+0000","last_clean":"2024-09-21T09:48:37.861618+0000","last_became_active":"2024-09-21T09:47:46.850604+0000","last_became_peered":"2024-09-21T09:47:46.850604+0000","last_unstale":"2024-09-21T09:48:37.861618+0000","last_undegraded":"2024-09-21T09:48:37.861618+0000","last_fullsized":"2024-09-21T09:48:37.861618+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080096+0000","last_change":"2024-09-21T09:47:24.064334+0000","last_active":"2024-09-21T09:48:38.080096+0000","last_peered":"2024-09-21T09:48:38.080096+0000","last_clean":"2024-09-21T09:48:38.080096+0000","last_became_active":"2024-09-21T09:47:24.064063+0000","last_became_peered":"2024-09-21T09:47:24.064063+0000","last_unstale":"2024-09-21T09:48:38.080096+0000","last_undegraded":"2024-09-21T09:48:38.080096+0000","last_fullsized":"2024-09-21T09:48:38.080096+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886822+0000","last_change":"2024-09-21T09:47:24.058842+0000","last_active":"2024-09-21T09:48:37.886822+0000","last_peered":"2024-09-21T09:48:37.886822+0000","last_clean":"2024-09-21T09:48:37.886822+0000","last_became_active":"2024-09-21T09:47:24.058245+0000","last_became_peered":"2024-09-21T09:47:24.058245+0000","last_unstale":"2024-09-21T09:48:37.886822+0000","last_undegraded":"2024-09-21T09:48:37.886822+0000","last_fullsized":"2024-09-21T09:48:37.886822+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860590+0000","last_change":"2024-09-21T09:47:24.052028+0000","last_active":"2024-09-21T09:48:37.860590+0000","last_peered":"2024-09-21T09:48:37.860590+0000","last_clean":"2024-09-21T09:48:37.860590+0000","last_became_active":"2024-09-21T09:47:24.051766+0000","last_became_peered":"2024-09-21T09:47:24.051766+0000","last_unstale":"2024-09-21T09:48:37.860590+0000","last_undegraded":"2024-09-21T09:48:37.860590+0000","last_fullsized":"2024-09-21T09:48:37.860590+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865236+0000","last_change":"2024-09-21T09:47:24.060117+0000","last_active":"2024-09-21T09:48:37.865236+0000","last_peered":"2024-09-21T09:48:37.865236+0000","last_clean":"2024-09-21T09:48:37.865236+0000","last_became_active":"2024-09-21T09:47:24.059852+0000","last_became_peered":"2024-09-21T09:47:24.059852+0000","last_unstale":"2024-09-21T09:48:37.865236+0000","last_undegraded":"2024-09-21T09:48:37.865236+0000","last_fullsized":"2024-09-21T09:48:37.865236+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865999+0000","last_change":"2024-09-21T09:47:46.480400+0000","last_active":"2024-09-21T09:48:37.865999+0000","last_peered":"2024-09-21T09:48:37.865999+0000","last_clean":"2024-09-21T09:48:37.865999+0000","last_became_active":"2024-09-21T09:47:46.480103+0000","last_became_peered":"2024-09-21T09:47:46.480103+0000","last_unstale":"2024-09-21T09:48:37.865999+0000","last_undegraded":"2024-09-21T09:48:37.865999+0000","last_fullsized":"2024-09-21T09:48:37.865999+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.862210+0000","last_change":"2024-09-21T09:47:24.057147+0000","last_active":"2024-09-21T09:48:37.862210+0000","last_peered":"2024-09-21T09:48:37.862210+0000","last_clean":"2024-09-21T09:48:37.862210+0000","last_became_active":"2024-09-21T09:47:24.056798+0000","last_became_peered":"2024-09-21T09:47:24.056798+0000","last_unstale":"2024-09-21T09:48:37.862210+0000","last_undegraded":"2024-09-21T09:48:37.862210+0000","last_fullsized":"2024-09-21T09:48:37.862210+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978661+0000","last_change":"2024-09-21T09:47:24.070640+0000","last_active":"2024-09-21T09:48:37.978661+0000","last_peered":"2024-09-21T09:48:37.978661+0000","last_clean":"2024-09-21T09:48:37.978661+0000","last_became_active":"2024-09-21T09:47:24.070359+0000","last_became_peered":"2024-09-21T09:47:24.070359+0000","last_unstale":"2024-09-21T09:48:37.978661+0000","last_undegraded":"2024-09-21T09:48:37.978661+0000","last_fullsized":"2024-09-21T09:48:37.978661+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080608+0000","last_change":"2024-09-21T09:47:24.056659+0000","last_active":"2024-09-21T09:48:38.080608+0000","last_peered":"2024-09-21T09:48:38.080608+0000","last_clean":"2024-09-21T09:48:38.080608+0000","last_became_active":"2024-09-21T09:47:24.056360+0000","last_became_peered":"2024-09-21T09:47:24.056360+0000","last_unstale":"2024-09-21T09:48:38.080608+0000","last_undegraded":"2024-09-21T09:48:38.080608+0000","last_fullsized":"2024-09-21T09:48:38.080608+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886909+0000","last_change":"2024-09-21T09:47:24.065154+0000","last_active":"2024-09-21T09:48:37.886909+0000","last_peered":"2024-09-21T09:48:37.886909+0000","last_clean":"2024-09-21T09:48:37.886909+0000","last_became_active":"2024-09-21T09:47:24.064231+0000","last_became_peered":"2024-09-21T09:47:24.064231+0000","last_unstale":"2024-09-21T09:48:37.886909+0000","last_undegraded":"2024-09-21T09:48:37.886909+0000","last_fullsized":"2024-09-21T09:48:37.886909+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865558+0000","last_change":"2024-09-21T09:47:24.062716+0000","last_active":"2024-09-21T09:48:37.865558+0000","last_peered":"2024-09-21T09:48:37.865558+0000","last_clean":"2024-09-21T09:48:37.865558+0000","last_became_active":"2024-09-21T09:47:24.062437+0000","last_became_peered":"2024-09-21T09:47:24.062437+0000","last_unstale":"2024-09-21T09:48:37.865558+0000","last_undegraded":"2024-09-21T09:48:37.865558+0000","last_fullsized":"2024-09-21T09:48:37.865558+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079839+0000","last_change":"2024-09-21T09:47:24.051387+0000","last_active":"2024-09-21T09:48:38.079839+0000","last_peered":"2024-09-21T09:48:38.079839+0000","last_clean":"2024-09-21T09:48:38.079839+0000","last_became_active":"2024-09-21T09:47:24.051090+0000","last_became_peered":"2024-09-21T09:47:24.051090+0000","last_unstale":"2024-09-21T09:48:38.079839+0000","last_undegraded":"2024-09-21T09:48:38.079839+0000","last_fullsized":"2024-09-21T09:48:38.079839+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865298+0000","last_change":"2024-09-21T09:47:24.058079+0000","last_active":"2024-09-21T09:48:37.865298+0000","last_peered":"2024-09-21T09:48:37.865298+0000","last_clean":"2024-09-21T09:48:37.865298+0000","last_became_active":"2024-09-21T09:47:24.057753+0000","last_became_peered":"2024-09-21T09:47:24.057753+0000","last_unstale":"2024-09-21T09:48:37.865298+0000","last_undegraded":"2024-09-21T09:48:37.865298+0000","last_fullsized":"2024-09-21T09:48:37.865298+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080292+0000","last_change":"2024-09-21T09:47:24.057220+0000","last_active":"2024-09-21T09:48:38.080292+0000","last_peered":"2024-09-21T09:48:38.080292+0000","last_clean":"2024-09-21T09:48:38.080292+0000","last_became_active":"2024-09-21T09:47:24.056915+0000","last_became_peered":"2024-09-21T09:47:24.056915+0000","last_unstale":"2024-09-21T09:48:38.080292+0000","last_undegraded":"2024-09-21T09:48:38.080292+0000","last_fullsized":"2024-09-21T09:48:38.080292+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080917+0000","last_change":"2024-09-21T09:47:24.057310+0000","last_active":"2024-09-21T09:48:38.080917+0000","last_peered":"2024-09-21T09:48:38.080917+0000","last_clean":"2024-09-21T09:48:38.080917+0000","last_became_active":"2024-09-21T09:47:24.057047+0000","last_became_peered":"2024-09-21T09:47:24.057047+0000","last_unstale":"2024-09-21T09:48:38.080917+0000","last_undegraded":"2024-09-21T09:48:38.080917+0000","last_fullsized":"2024-09-21T09:48:38.080917+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861084+0000","last_change":"2024-09-21T09:47:24.060976+0000","last_active":"2024-09-21T09:48:37.861084+0000","last_peered":"2024-09-21T09:48:37.861084+0000","last_clean":"2024-09-21T09:48:37.861084+0000","last_became_active":"2024-09-21T09:47:24.060725+0000","last_became_peered":"2024-09-21T09:47:24.060725+0000","last_unstale":"2024-09-21T09:48:37.861084+0000","last_undegraded":"2024-09-21T09:48:37.861084+0000","last_fullsized":"2024-09-21T09:48:37.861084+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080326+0000","last_change":"2024-09-21T09:47:24.053563+0000","last_active":"2024-09-21T09:48:38.080326+0000","last_peered":"2024-09-21T09:48:38.080326+0000","last_clean":"2024-09-21T09:48:38.080326+0000","last_became_active":"2024-09-21T09:47:24.052853+0000","last_became_peered":"2024-09-21T09:47:24.052853+0000","last_unstale":"2024-09-21T09:48:38.080326+0000","last_undegraded":"2024-09-21T09:48:38.080326+0000","last_fullsized":"2024-09-21T09:48:38.080326+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886944+0000","last_change":"2024-09-21T09:47:24.068204+0000","last_active":"2024-09-21T09:48:37.886944+0000","last_peered":"2024-09-21T09:48:37.886944+0000","last_clean":"2024-09-21T09:48:37.886944+0000","last_became_active":"2024-09-21T09:47:24.067911+0000","last_became_peered":"2024-09-21T09:47:24.067911+0000","last_unstale":"2024-09-21T09:48:37.886944+0000","last_undegraded":"2024-09-21T09:48:37.886944+0000","last_fullsized":"2024-09-21T09:48:37.886944+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865668+0000","last_change":"2024-09-21T09:47:24.058882+0000","last_active":"2024-09-21T09:48:37.865668+0000","last_peered":"2024-09-21T09:48:37.865668+0000","last_clean":"2024-09-21T09:48:37.865668+0000","last_became_active":"2024-09-21T09:47:24.058587+0000","last_became_peered":"2024-09-21T09:47:24.058587+0000","last_unstale":"2024-09-21T09:48:37.865668+0000","last_undegraded":"2024-09-21T09:48:37.865668+0000","last_fullsized":"2024-09-21T09:48:37.865668+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861701+0000","last_change":"2024-09-21T09:47:24.060446+0000","last_active":"2024-09-21T09:48:37.861701+0000","last_peered":"2024-09-21T09:48:37.861701+0000","last_clean":"2024-09-21T09:48:37.861701+0000","last_became_active":"2024-09-21T09:47:24.060165+0000","last_became_peered":"2024-09-21T09:47:24.060165+0000","last_unstale":"2024-09-21T09:48:37.861701+0000","last_undegraded":"2024-09-21T09:48:37.861701+0000","last_fullsized":"2024-09-21T09:48:37.861701+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080189+0000","last_change":"2024-09-21T09:47:24.059974+0000","last_active":"2024-09-21T09:48:38.080189+0000","last_peered":"2024-09-21T09:48:38.080189+0000","last_clean":"2024-09-21T09:48:38.080189+0000","last_became_active":"2024-09-21T09:47:24.059270+0000","last_became_peered":"2024-09-21T09:47:24.059270+0000","last_unstale":"2024-09-21T09:48:38.080189+0000","last_undegraded":"2024-09-21T09:48:38.080189+0000","last_fullsized":"2024-09-21T09:48:38.080189+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.859975+0000","last_change":"2024-09-21T09:47:24.048941+0000","last_active":"2024-09-21T09:48:37.859975+0000","last_peered":"2024-09-21T09:48:37.859975+0000","last_clean":"2024-09-21T09:48:37.859975+0000","last_became_active":"2024-09-21T09:47:24.048631+0000","last_became_peered":"2024-09-21T09:47:24.048631+0000","last_unstale":"2024-09-21T09:48:37.859975+0000","last_undegraded":"2024-09-21T09:48:37.859975+0000","last_fullsized":"2024-09-21T09:48:37.859975+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080417+0000","last_change":"2024-09-21T09:47:20.866010+0000","last_active":"2024-09-21T09:48:38.080417+0000","last_peered":"2024-09-21T09:48:38.080417+0000","last_clean":"2024-09-21T09:48:38.080417+0000","last_became_active":"2024-09-21T09:47:20.865732+0000","last_became_peered":"2024-09-21T09:47:20.865732+0000","last_unstale":"2024-09-21T09:48:38.080417+0000","last_undegraded":"2024-09-21T09:48:38.080417+0000","last_fullsized":"2024-09-21T09:48:38.080417+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886995+0000","last_change":"2024-09-21T09:47:24.067813+0000","last_active":"2024-09-21T09:48:37.886995+0000","last_peered":"2024-09-21T09:48:37.886995+0000","last_clean":"2024-09-21T09:48:37.886995+0000","last_became_active":"2024-09-21T09:47:24.067412+0000","last_became_peered":"2024-09-21T09:47:24.067412+0000","last_unstale":"2024-09-21T09:48:37.886995+0000","last_undegraded":"2024-09-21T09:48:37.886995+0000","last_fullsized":"2024-09-21T09:48:37.886995+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":35,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080597+0000","last_change":"2024-09-21T09:47:46.481320+0000","last_active":"2024-09-21T09:48:38.080597+0000","last_peered":"2024-09-21T09:48:38.080597+0000","last_clean":"2024-09-21T09:48:38.080597+0000","last_became_active":"2024-09-21T09:47:46.480905+0000","last_became_peered":"2024-09-21T09:47:46.480905+0000","last_unstale":"2024-09-21T09:48:38.080597+0000","last_undegraded":"2024-09-21T09:48:38.080597+0000","last_fullsized":"2024-09-21T09:48:38.080597+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:23.831301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079961+0000","last_change":"2024-09-21T09:47:20.859829+0000","last_active":"2024-09-21T09:48:38.079961+0000","last_peered":"2024-09-21T09:48:38.079961+0000","last_clean":"2024-09-21T09:48:38.079961+0000","last_became_active":"2024-09-21T09:47:20.859507+0000","last_became_peered":"2024-09-21T09:47:20.859507+0000","last_unstale":"2024-09-21T09:48:38.079961+0000","last_undegraded":"2024-09-21T09:48:38.079961+0000","last_fullsized":"2024-09-21T09:48:38.079961+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079902+0000","last_change":"2024-09-21T09:47:24.053702+0000","last_active":"2024-09-21T09:48:38.079902+0000","last_peered":"2024-09-21T09:48:38.079902+0000","last_clean":"2024-09-21T09:48:38.079902+0000","last_became_active":"2024-09-21T09:47:24.053295+0000","last_became_peered":"2024-09-21T09:47:24.053295+0000","last_unstale":"2024-09-21T09:48:38.079902+0000","last_undegraded":"2024-09-21T09:48:38.079902+0000","last_fullsized":"2024-09-21T09:48:38.079902+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865313+0000","last_change":"2024-09-21T09:47:20.863181+0000","last_active":"2024-09-21T09:48:37.865313+0000","last_peered":"2024-09-21T09:48:37.865313+0000","last_clean":"2024-09-21T09:48:37.865313+0000","last_became_active":"2024-09-21T09:47:20.862895+0000","last_became_peered":"2024-09-21T09:47:20.862895+0000","last_unstale":"2024-09-21T09:48:37.865313+0000","last_undegraded":"2024-09-21T09:48:37.865313+0000","last_fullsized":"2024-09-21T09:48:37.865313+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860485+0000","last_change":"2024-09-21T09:47:24.051248+0000","last_active":"2024-09-21T09:48:37.860485+0000","last_peered":"2024-09-21T09:48:37.860485+0000","last_clean":"2024-09-21T09:48:37.860485+0000","last_became_active":"2024-09-21T09:47:24.050918+0000","last_became_peered":"2024-09-21T09:47:24.050918+0000","last_unstale":"2024-09-21T09:48:37.860485+0000","last_undegraded":"2024-09-21T09:48:37.860485+0000","last_fullsized":"2024-09-21T09:48:37.860485+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080973+0000","last_change":"2024-09-21T09:47:20.865532+0000","last_active":"2024-09-21T09:48:38.080973+0000","last_peered":"2024-09-21T09:48:38.080973+0000","last_clean":"2024-09-21T09:48:38.080973+0000","last_became_active":"2024-09-21T09:47:20.865103+0000","last_became_peered":"2024-09-21T09:47:20.865103+0000","last_unstale":"2024-09-21T09:48:38.080973+0000","last_undegraded":"2024-09-21T09:48:38.080973+0000","last_fullsized":"2024-09-21T09:48:38.080973+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865804+0000","last_change":"2024-09-21T09:47:24.067330+0000","last_active":"2024-09-21T09:48:37.865804+0000","last_peered":"2024-09-21T09:48:37.865804+0000","last_clean":"2024-09-21T09:48:37.865804+0000","last_became_active":"2024-09-21T09:47:24.066939+0000","last_became_peered":"2024-09-21T09:47:24.066939+0000","last_unstale":"2024-09-21T09:48:37.865804+0000","last_undegraded":"2024-09-21T09:48:37.865804+0000","last_fullsized":"2024-09-21T09:48:37.865804+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865951+0000","last_change":"2024-09-21T09:47:20.860605+0000","last_active":"2024-09-21T09:48:37.865951+0000","last_peered":"2024-09-21T09:48:37.865951+0000","last_clean":"2024-09-21T09:48:37.865951+0000","last_became_active":"2024-09-21T09:47:20.860288+0000","last_became_peered":"2024-09-21T09:47:20.860288+0000","last_unstale":"2024-09-21T09:48:37.865951+0000","last_undegraded":"2024-09-21T09:48:37.865951+0000","last_fullsized":"2024-09-21T09:48:37.865951+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887587+0000","last_change":"2024-09-21T09:47:24.057860+0000","last_active":"2024-09-21T09:48:37.887587+0000","last_peered":"2024-09-21T09:48:37.887587+0000","last_clean":"2024-09-21T09:48:37.887587+0000","last_became_active":"2024-09-21T09:47:24.057594+0000","last_became_peered":"2024-09-21T09:47:24.057594+0000","last_unstale":"2024-09-21T09:48:37.887587+0000","last_undegraded":"2024-09-21T09:48:37.887587+0000","last_fullsized":"2024-09-21T09:48:37.887587+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860960+0000","last_change":"2024-09-21T09:47:20.857026+0000","last_active":"2024-09-21T09:48:37.860960+0000","last_peered":"2024-09-21T09:48:37.860960+0000","last_clean":"2024-09-21T09:48:37.860960+0000","last_became_active":"2024-09-21T09:47:20.856719+0000","last_became_peered":"2024-09-21T09:47:20.856719+0000","last_unstale":"2024-09-21T09:48:37.860960+0000","last_undegraded":"2024-09-21T09:48:37.860960+0000","last_fullsized":"2024-09-21T09:48:37.860960+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860234+0000","last_change":"2024-09-21T09:47:24.059537+0000","last_active":"2024-09-21T09:48:37.860234+0000","last_peered":"2024-09-21T09:48:37.860234+0000","last_clean":"2024-09-21T09:48:37.860234+0000","last_became_active":"2024-09-21T09:47:24.059264+0000","last_became_peered":"2024-09-21T09:47:24.059264+0000","last_unstale":"2024-09-21T09:48:37.860234+0000","last_undegraded":"2024-09-21T09:48:37.860234+0000","last_fullsized":"2024-09-21T09:48:37.860234+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"60'9","reported_seq":33,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080662+0000","last_change":"2024-09-21T09:47:20.864446+0000","last_active":"2024-09-21T09:48:38.080662+0000","last_peered":"2024-09-21T09:48:38.080662+0000","last_clean":"2024-09-21T09:48:38.080662+0000","last_became_active":"2024-09-21T09:47:20.864160+0000","last_became_peered":"2024-09-21T09:47:20.864160+0000","last_unstale":"2024-09-21T09:48:38.080662+0000","last_undegraded":"2024-09-21T09:48:38.080662+0000","last_fullsized":"2024-09-21T09:48:38.080662+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978421+0000","last_change":"2024-09-21T09:47:24.070010+0000","last_active":"2024-09-21T09:48:37.978421+0000","last_peered":"2024-09-21T09:48:37.978421+0000","last_clean":"2024-09-21T09:48:37.978421+0000","last_became_active":"2024-09-21T09:47:24.069698+0000","last_became_peered":"2024-09-21T09:47:24.069698+0000","last_unstale":"2024-09-21T09:48:37.978421+0000","last_undegraded":"2024-09-21T09:48:37.978421+0000","last_fullsized":"2024-09-21T09:48:37.978421+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887044+0000","last_change":"2024-09-21T09:47:20.858242+0000","last_active":"2024-09-21T09:48:37.887044+0000","last_peered":"2024-09-21T09:48:37.887044+0000","last_clean":"2024-09-21T09:48:37.887044+0000","last_became_active":"2024-09-21T09:47:20.857951+0000","last_became_peered":"2024-09-21T09:47:20.857951+0000","last_unstale":"2024-09-21T09:48:37.887044+0000","last_undegraded":"2024-09-21T09:48:37.887044+0000","last_fullsized":"2024-09-21T09:48:37.887044+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886995+0000","last_change":"2024-09-21T09:47:24.063666+0000","last_active":"2024-09-21T09:48:37.886995+0000","last_peered":"2024-09-21T09:48:37.886995+0000","last_clean":"2024-09-21T09:48:37.886995+0000","last_became_active":"2024-09-21T09:47:24.063116+0000","last_became_peered":"2024-09-21T09:47:24.063116+0000","last_unstale":"2024-09-21T09:48:37.886995+0000","last_undegraded":"2024-09-21T09:48:37.886995+0000","last_fullsized":"2024-09-21T09:48:37.886995+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865774+0000","last_change":"2024-09-21T09:47:20.867134+0000","last_active":"2024-09-21T09:48:37.865774+0000","last_peered":"2024-09-21T09:48:37.865774+0000","last_clean":"2024-09-21T09:48:37.865774+0000","last_became_active":"2024-09-21T09:47:20.866875+0000","last_became_peered":"2024-09-21T09:47:20.866875+0000","last_unstale":"2024-09-21T09:48:37.865774+0000","last_undegraded":"2024-09-21T09:48:37.865774+0000","last_fullsized":"2024-09-21T09:48:37.865774+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978277+0000","last_change":"2024-09-21T09:47:24.069831+0000","last_active":"2024-09-21T09:48:37.978277+0000","last_peered":"2024-09-21T09:48:37.978277+0000","last_clean":"2024-09-21T09:48:37.978277+0000","last_became_active":"2024-09-21T09:47:24.069569+0000","last_became_peered":"2024-09-21T09:47:24.069569+0000","last_unstale":"2024-09-21T09:48:37.978277+0000","last_undegraded":"2024-09-21T09:48:37.978277+0000","last_fullsized":"2024-09-21T09:48:37.978277+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080355+0000","last_change":"2024-09-21T09:47:20.862992+0000","last_active":"2024-09-21T09:48:38.080355+0000","last_peered":"2024-09-21T09:48:38.080355+0000","last_clean":"2024-09-21T09:48:38.080355+0000","last_became_active":"2024-09-21T09:47:20.862582+0000","last_became_peered":"2024-09-21T09:47:20.862582+0000","last_unstale":"2024-09-21T09:48:38.080355+0000","last_undegraded":"2024-09-21T09:48:38.080355+0000","last_fullsized":"2024-09-21T09:48:38.080355+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080230+0000","last_change":"2024-09-21T09:47:24.058311+0000","last_active":"2024-09-21T09:48:38.080230+0000","last_peered":"2024-09-21T09:48:38.080230+0000","last_clean":"2024-09-21T09:48:38.080230+0000","last_became_active":"2024-09-21T09:47:24.057961+0000","last_became_peered":"2024-09-21T09:47:24.057961+0000","last_unstale":"2024-09-21T09:48:38.080230+0000","last_undegraded":"2024-09-21T09:48:38.080230+0000","last_fullsized":"2024-09-21T09:48:38.080230+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":32,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860182+0000","last_change":"2024-09-21T09:47:20.856509+0000","last_active":"2024-09-21T09:48:37.860182+0000","last_peered":"2024-09-21T09:48:37.860182+0000","last_clean":"2024-09-21T09:48:37.860182+0000","last_became_active":"2024-09-21T09:47:20.856229+0000","last_became_peered":"2024-09-21T09:47:20.856229+0000","last_unstale":"2024-09-21T09:48:37.860182+0000","last_undegraded":"2024-09-21T09:48:37.860182+0000","last_fullsized":"2024-09-21T09:48:37.860182+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978178+0000","last_change":"2024-09-21T09:47:24.069301+0000","last_active":"2024-09-21T09:48:37.978178+0000","last_peered":"2024-09-21T09:48:37.978178+0000","last_clean":"2024-09-21T09:48:37.978178+0000","last_became_active":"2024-09-21T09:47:24.068917+0000","last_became_peered":"2024-09-21T09:47:24.068917+0000","last_unstale":"2024-09-21T09:48:37.978178+0000","last_undegraded":"2024-09-21T09:48:37.978178+0000","last_fullsized":"2024-09-21T09:48:37.978178+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860611+0000","last_change":"2024-09-21T09:47:20.862118+0000","last_active":"2024-09-21T09:48:37.860611+0000","last_peered":"2024-09-21T09:48:37.860611+0000","last_clean":"2024-09-21T09:48:37.860611+0000","last_became_active":"2024-09-21T09:47:20.861815+0000","last_became_peered":"2024-09-21T09:47:20.861815+0000","last_unstale":"2024-09-21T09:48:37.860611+0000","last_undegraded":"2024-09-21T09:48:37.860611+0000","last_fullsized":"2024-09-21T09:48:37.860611+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978644+0000","last_change":"2024-09-21T09:47:24.069502+0000","last_active":"2024-09-21T09:48:37.978644+0000","last_peered":"2024-09-21T09:48:37.978644+0000","last_clean":"2024-09-21T09:48:37.978644+0000","last_became_active":"2024-09-21T09:47:24.069235+0000","last_became_peered":"2024-09-21T09:47:24.069235+0000","last_unstale":"2024-09-21T09:48:37.978644+0000","last_undegraded":"2024-09-21T09:48:37.978644+0000","last_fullsized":"2024-09-21T09:48:37.978644+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.862390+0000","last_change":"2024-09-21T09:47:20.862503+0000","last_active":"2024-09-21T09:48:37.862390+0000","last_peered":"2024-09-21T09:48:37.862390+0000","last_clean":"2024-09-21T09:48:37.862390+0000","last_became_active":"2024-09-21T09:47:20.862237+0000","last_became_peered":"2024-09-21T09:47:20.862237+0000","last_unstale":"2024-09-21T09:48:37.862390+0000","last_undegraded":"2024-09-21T09:48:37.862390+0000","last_fullsized":"2024-09-21T09:48:37.862390+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978686+0000","last_change":"2024-09-21T09:47:24.069273+0000","last_active":"2024-09-21T09:48:37.978686+0000","last_peered":"2024-09-21T09:48:37.978686+0000","last_clean":"2024-09-21T09:48:37.978686+0000","last_became_active":"2024-09-21T09:47:24.068990+0000","last_became_peered":"2024-09-21T09:47:24.068990+0000","last_unstale":"2024-09-21T09:48:37.978686+0000","last_undegraded":"2024-09-21T09:48:37.978686+0000","last_fullsized":"2024-09-21T09:48:37.978686+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861047+0000","last_change":"2024-09-21T09:47:20.858095+0000","last_active":"2024-09-21T09:48:37.861047+0000","last_peered":"2024-09-21T09:48:37.861047+0000","last_clean":"2024-09-21T09:48:37.861047+0000","last_became_active":"2024-09-21T09:47:20.857760+0000","last_became_peered":"2024-09-21T09:47:20.857760+0000","last_unstale":"2024-09-21T09:48:37.861047+0000","last_undegraded":"2024-09-21T09:48:37.861047+0000","last_fullsized":"2024-09-21T09:48:37.861047+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865642+0000","last_change":"2024-09-21T09:47:24.063342+0000","last_active":"2024-09-21T09:48:37.865642+0000","last_peered":"2024-09-21T09:48:37.865642+0000","last_clean":"2024-09-21T09:48:37.865642+0000","last_became_active":"2024-09-21T09:47:24.063070+0000","last_became_peered":"2024-09-21T09:47:24.063070+0000","last_unstale":"2024-09-21T09:48:37.865642+0000","last_undegraded":"2024-09-21T09:48:37.865642+0000","last_fullsized":"2024-09-21T09:48:37.865642+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861853+0000","last_change":"2024-09-21T09:47:20.859702+0000","last_active":"2024-09-21T09:48:37.861853+0000","last_peered":"2024-09-21T09:48:37.861853+0000","last_clean":"2024-09-21T09:48:37.861853+0000","last_became_active":"2024-09-21T09:47:20.859443+0000","last_became_peered":"2024-09-21T09:47:20.859443+0000","last_unstale":"2024-09-21T09:48:37.861853+0000","last_undegraded":"2024-09-21T09:48:37.861853+0000","last_fullsized":"2024-09-21T09:48:37.861853+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978522+0000","last_change":"2024-09-21T09:47:24.070338+0000","last_active":"2024-09-21T09:48:37.978522+0000","last_peered":"2024-09-21T09:48:37.978522+0000","last_clean":"2024-09-21T09:48:37.978522+0000","last_became_active":"2024-09-21T09:47:24.070074+0000","last_became_peered":"2024-09-21T09:47:24.070074+0000","last_unstale":"2024-09-21T09:48:37.978522+0000","last_undegraded":"2024-09-21T09:48:37.978522+0000","last_fullsized":"2024-09-21T09:48:37.978522+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.656763+0000","last_change":"2024-09-21T09:48:38.081026+0000","last_active":"2024-09-21T09:48:41.656763+0000","last_peered":"2024-09-21T09:48:41.656763+0000","last_clean":"2024-09-21T09:48:41.656763+0000","last_became_active":"2024-09-21T09:48:33.296475+0000","last_became_peered":"2024-09-21T09:48:33.296475+0000","last_unstale":"2024-09-21T09:48:41.656763+0000","last_undegraded":"2024-09-21T09:48:41.656763+0000","last_fullsized":"2024-09-21T09:48:41.656763+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:15.020915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046213800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"60'79","reported_seq":58,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978200+0000","last_change":"2024-09-21T09:46:57.426595+0000","last_active":"2024-09-21T09:48:37.978200+0000","last_peered":"2024-09-21T09:48:37.978200+0000","last_clean":"2024-09-21T09:48:37.978200+0000","last_became_active":"2024-09-21T09:46:57.418997+0000","last_became_peered":"2024-09-21T09:46:57.418997+0000","last_unstale":"2024-09-21T09:48:37.978200+0000","last_undegraded":"2024-09-21T09:48:37.978200+0000","last_fullsized":"2024-09-21T09:48:37.978200+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_clean_scrub_stamp":"2024-09-21T09:44:57.819716+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865760+0000","last_change":"2024-09-21T09:47:20.866500+0000","last_active":"2024-09-21T09:48:37.865760+0000","last_peered":"2024-09-21T09:48:37.865760+0000","last_clean":"2024-09-21T09:48:37.865760+0000","last_became_active":"2024-09-21T09:47:20.866224+0000","last_became_peered":"2024-09-21T09:47:20.866224+0000","last_unstale":"2024-09-21T09:48:37.865760+0000","last_undegraded":"2024-09-21T09:48:37.865760+0000","last_fullsized":"2024-09-21T09:48:37.865760+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079954+0000","last_change":"2024-09-21T09:47:24.061053+0000","last_active":"2024-09-21T09:48:38.079954+0000","last_peered":"2024-09-21T09:48:38.079954+0000","last_clean":"2024-09-21T09:48:38.079954+0000","last_became_active":"2024-09-21T09:47:24.060632+0000","last_became_peered":"2024-09-21T09:47:24.060632+0000","last_unstale":"2024-09-21T09:48:38.079954+0000","last_undegraded":"2024-09-21T09:48:38.079954+0000","last_fullsized":"2024-09-21T09:48:38.079954+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.509038+0000","last_change":"2024-09-21T09:48:37.887537+0000","last_active":"2024-09-21T09:48:41.509038+0000","last_peered":"2024-09-21T09:48:41.509038+0000","last_clean":"2024-09-21T09:48:41.509038+0000","last_became_active":"2024-09-21T09:48:33.298887+0000","last_became_peered":"2024-09-21T09:48:33.298887+0000","last_unstale":"2024-09-21T09:48:41.509038+0000","last_undegraded":"2024-09-21T09:48:41.509038+0000","last_fullsized":"2024-09-21T09:48:41.509038+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:16.078381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027730799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978152+0000","last_change":"2024-09-21T09:47:20.869192+0000","last_active":"2024-09-21T09:48:37.978152+0000","last_peered":"2024-09-21T09:48:37.978152+0000","last_clean":"2024-09-21T09:48:37.978152+0000","last_became_active":"2024-09-21T09:47:20.868800+0000","last_became_peered":"2024-09-21T09:47:20.868800+0000","last_unstale":"2024-09-21T09:48:37.978152+0000","last_undegraded":"2024-09-21T09:48:37.978152+0000","last_fullsized":"2024-09-21T09:48:37.978152+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886738+0000","last_change":"2024-09-21T09:47:24.056658+0000","last_active":"2024-09-21T09:48:37.886738+0000","last_peered":"2024-09-21T09:48:37.886738+0000","last_clean":"2024-09-21T09:48:37.886738+0000","last_became_active":"2024-09-21T09:47:24.056366+0000","last_became_peered":"2024-09-21T09:47:24.056366+0000","last_unstale":"2024-09-21T09:48:37.886738+0000","last_undegraded":"2024-09-21T09:48:37.886738+0000","last_fullsized":"2024-09-21T09:48:37.886738+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.427162+0000","last_change":"2024-09-21T09:48:37.861462+0000","last_active":"2024-09-21T09:48:41.427162+0000","last_peered":"2024-09-21T09:48:41.427162+0000","last_clean":"2024-09-21T09:48:41.427162+0000","last_became_active":"2024-09-21T09:48:33.677383+0000","last_became_peered":"2024-09-21T09:48:33.677383+0000","last_unstale":"2024-09-21T09:48:41.427162+0000","last_undegraded":"2024-09-21T09:48:41.427162+0000","last_fullsized":"2024-09-21T09:48:41.427162+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:12.106483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000471278,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861820+0000","last_change":"2024-09-21T09:47:20.861899+0000","last_active":"2024-09-21T09:48:37.861820+0000","last_peered":"2024-09-21T09:48:37.861820+0000","last_clean":"2024-09-21T09:48:37.861820+0000","last_became_active":"2024-09-21T09:47:20.861684+0000","last_became_peered":"2024-09-21T09:47:20.861684+0000","last_unstale":"2024-09-21T09:48:37.861820+0000","last_undegraded":"2024-09-21T09:48:37.861820+0000","last_fullsized":"2024-09-21T09:48:37.861820+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860233+0000","last_change":"2024-09-21T09:47:24.058235+0000","last_active":"2024-09-21T09:48:37.860233+0000","last_peered":"2024-09-21T09:48:37.860233+0000","last_clean":"2024-09-21T09:48:37.860233+0000","last_became_active":"2024-09-21T09:47:24.057972+0000","last_became_peered":"2024-09-21T09:47:24.057972+0000","last_unstale":"2024-09-21T09:48:37.860233+0000","last_undegraded":"2024-09-21T09:48:37.860233+0000","last_fullsized":"2024-09-21T09:48:37.860233+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.427876+0000","last_change":"2024-09-21T09:48:37.861803+0000","last_active":"2024-09-21T09:48:41.427876+0000","last_peered":"2024-09-21T09:48:41.427876+0000","last_clean":"2024-09-21T09:48:41.427876+0000","last_became_active":"2024-09-21T09:48:33.295964+0000","last_became_peered":"2024-09-21T09:48:33.295964+0000","last_unstale":"2024-09-21T09:48:41.427876+0000","last_undegraded":"2024-09-21T09:48:41.427876+0000","last_fullsized":"2024-09-21T09:48:41.427876+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.417883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010536689999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865760+0000","last_change":"2024-09-21T09:47:20.861547+0000","last_active":"2024-09-21T09:48:37.865760+0000","last_peered":"2024-09-21T09:48:37.865760+0000","last_clean":"2024-09-21T09:48:37.865760+0000","last_became_active":"2024-09-21T09:47:20.861254+0000","last_became_peered":"2024-09-21T09:47:20.861254+0000","last_unstale":"2024-09-21T09:48:37.865760+0000","last_undegraded":"2024-09-21T09:48:37.865760+0000","last_fullsized":"2024-09-21T09:48:37.865760+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860142+0000","last_change":"2024-09-21T09:47:24.060191+0000","last_active":"2024-09-21T09:48:37.860142+0000","last_peered":"2024-09-21T09:48:37.860142+0000","last_clean":"2024-09-21T09:48:37.860142+0000","last_became_active":"2024-09-21T09:47:24.059929+0000","last_became_peered":"2024-09-21T09:47:24.059929+0000","last_unstale":"2024-09-21T09:48:37.860142+0000","last_undegraded":"2024-09-21T09:48:37.860142+0000","last_fullsized":"2024-09-21T09:48:37.860142+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"66'2","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.427101+0000","last_change":"2024-09-21T09:48:37.861137+0000","last_active":"2024-09-21T09:48:41.427101+0000","last_peered":"2024-09-21T09:48:41.427101+0000","last_clean":"2024-09-21T09:48:41.427101+0000","last_became_active":"2024-09-21T09:48:33.677309+0000","last_became_peered":"2024-09-21T09:48:33.677309+0000","last_unstale":"2024-09-21T09:48:41.427101+0000","last_undegraded":"2024-09-21T09:48:41.427101+0000","last_fullsized":"2024-09-21T09:48:41.427101+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:29.622677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056908600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861250+0000","last_change":"2024-09-21T09:47:20.862181+0000","last_active":"2024-09-21T09:48:37.861250+0000","last_peered":"2024-09-21T09:48:37.861250+0000","last_clean":"2024-09-21T09:48:37.861250+0000","last_became_active":"2024-09-21T09:47:20.861903+0000","last_became_peered":"2024-09-21T09:47:20.861903+0000","last_unstale":"2024-09-21T09:48:37.861250+0000","last_undegraded":"2024-09-21T09:48:37.861250+0000","last_fullsized":"2024-09-21T09:48:37.861250+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865932+0000","last_change":"2024-09-21T09:47:24.058849+0000","last_active":"2024-09-21T09:48:37.865932+0000","last_peered":"2024-09-21T09:48:37.865932+0000","last_clean":"2024-09-21T09:48:37.865932+0000","last_became_active":"2024-09-21T09:47:24.058564+0000","last_became_peered":"2024-09-21T09:47:24.058564+0000","last_unstale":"2024-09-21T09:48:37.865932+0000","last_undegraded":"2024-09-21T09:48:37.865932+0000","last_fullsized":"2024-09-21T09:48:37.865932+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"64'1","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.508707+0000","last_change":"2024-09-21T09:48:37.887211+0000","last_active":"2024-09-21T09:48:41.508707+0000","last_peered":"2024-09-21T09:48:41.508707+0000","last_clean":"2024-09-21T09:48:41.508707+0000","last_became_active":"2024-09-21T09:48:33.677589+0000","last_became_peered":"2024-09-21T09:48:33.677589+0000","last_unstale":"2024-09-21T09:48:41.508707+0000","last_undegraded":"2024-09-21T09:48:41.508707+0000","last_fullsized":"2024-09-21T09:48:41.508707+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:20.123742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040314399999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978674+0000","last_change":"2024-09-21T09:47:20.866885+0000","last_active":"2024-09-21T09:48:37.978674+0000","last_peered":"2024-09-21T09:48:37.978674+0000","last_clean":"2024-09-21T09:48:37.978674+0000","last_became_active":"2024-09-21T09:47:20.866436+0000","last_became_peered":"2024-09-21T09:47:20.866436+0000","last_unstale":"2024-09-21T09:48:37.978674+0000","last_undegraded":"2024-09-21T09:48:37.978674+0000","last_fullsized":"2024-09-21T09:48:37.978674+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861149+0000","last_change":"2024-09-21T09:47:24.060367+0000","last_active":"2024-09-21T09:48:37.861149+0000","last_peered":"2024-09-21T09:48:37.861149+0000","last_clean":"2024-09-21T09:48:37.861149+0000","last_became_active":"2024-09-21T09:47:24.060083+0000","last_became_peered":"2024-09-21T09:47:24.060083+0000","last_unstale":"2024-09-21T09:48:37.861149+0000","last_undegraded":"2024-09-21T09:48:37.861149+0000","last_fullsized":"2024-09-21T09:48:37.861149+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.655198+0000","last_change":"2024-09-21T09:48:38.080316+0000","last_active":"2024-09-21T09:48:41.655198+0000","last_peered":"2024-09-21T09:48:41.655198+0000","last_clean":"2024-09-21T09:48:41.655198+0000","last_became_active":"2024-09-21T09:48:33.298926+0000","last_became_peered":"2024-09-21T09:48:33.298926+0000","last_unstale":"2024-09-21T09:48:41.655198+0000","last_undegraded":"2024-09-21T09:48:41.655198+0000","last_fullsized":"2024-09-21T09:48:41.655198+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:34.343076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055753599999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861460+0000","last_change":"2024-09-21T09:47:20.858947+0000","last_active":"2024-09-21T09:48:37.861460+0000","last_peered":"2024-09-21T09:48:37.861460+0000","last_clean":"2024-09-21T09:48:37.861460+0000","last_became_active":"2024-09-21T09:47:20.858680+0000","last_became_peered":"2024-09-21T09:47:20.858680+0000","last_unstale":"2024-09-21T09:48:37.861460+0000","last_undegraded":"2024-09-21T09:48:37.861460+0000","last_fullsized":"2024-09-21T09:48:37.861460+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080299+0000","last_change":"2024-09-21T09:47:24.060498+0000","last_active":"2024-09-21T09:48:38.080299+0000","last_peered":"2024-09-21T09:48:38.080299+0000","last_clean":"2024-09-21T09:48:38.080299+0000","last_became_active":"2024-09-21T09:47:24.060179+0000","last_became_peered":"2024-09-21T09:47:24.060179+0000","last_unstale":"2024-09-21T09:48:38.080299+0000","last_undegraded":"2024-09-21T09:48:38.080299+0000","last_fullsized":"2024-09-21T09:48:38.080299+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.655530+0000","last_change":"2024-09-21T09:48:37.860425+0000","last_active":"2024-09-21T09:48:41.655530+0000","last_peered":"2024-09-21T09:48:41.655530+0000","last_clean":"2024-09-21T09:48:41.655530+0000","last_became_active":"2024-09-21T09:48:33.677625+0000","last_became_peered":"2024-09-21T09:48:33.677625+0000","last_unstale":"2024-09-21T09:48:41.655530+0000","last_undegraded":"2024-09-21T09:48:41.655530+0000","last_fullsized":"2024-09-21T09:48:41.655530+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:51.936200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069840600000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.081020+0000","last_change":"2024-09-21T09:47:20.865797+0000","last_active":"2024-09-21T09:48:38.081020+0000","last_peered":"2024-09-21T09:48:38.081020+0000","last_clean":"2024-09-21T09:48:38.081020+0000","last_became_active":"2024-09-21T09:47:20.865466+0000","last_became_peered":"2024-09-21T09:47:20.865466+0000","last_unstale":"2024-09-21T09:48:38.081020+0000","last_undegraded":"2024-09-21T09:48:38.081020+0000","last_fullsized":"2024-09-21T09:48:38.081020+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887191+0000","last_change":"2024-09-21T09:47:24.058690+0000","last_active":"2024-09-21T09:48:37.887191+0000","last_peered":"2024-09-21T09:48:37.887191+0000","last_clean":"2024-09-21T09:48:37.887191+0000","last_became_active":"2024-09-21T09:47:24.058371+0000","last_became_peered":"2024-09-21T09:47:24.058371+0000","last_unstale":"2024-09-21T09:48:37.887191+0000","last_undegraded":"2024-09-21T09:48:37.887191+0000","last_fullsized":"2024-09-21T09:48:37.887191+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978109+0000","last_change":"2024-09-21T09:47:20.868994+0000","last_active":"2024-09-21T09:48:37.978109+0000","last_peered":"2024-09-21T09:48:37.978109+0000","last_clean":"2024-09-21T09:48:37.978109+0000","last_became_active":"2024-09-21T09:47:20.868686+0000","last_became_peered":"2024-09-21T09:47:20.868686+0000","last_unstale":"2024-09-21T09:48:37.978109+0000","last_undegraded":"2024-09-21T09:48:37.978109+0000","last_fullsized":"2024-09-21T09:48:37.978109+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860137+0000","last_change":"2024-09-21T09:47:24.055068+0000","last_active":"2024-09-21T09:48:37.860137+0000","last_peered":"2024-09-21T09:48:37.860137+0000","last_clean":"2024-09-21T09:48:37.860137+0000","last_became_active":"2024-09-21T09:47:24.054785+0000","last_became_peered":"2024-09-21T09:47:24.054785+0000","last_unstale":"2024-09-21T09:48:37.860137+0000","last_undegraded":"2024-09-21T09:48:37.860137+0000","last_fullsized":"2024-09-21T09:48:37.860137+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860615+0000","last_change":"2024-09-21T09:47:20.862674+0000","last_active":"2024-09-21T09:48:37.860615+0000","last_peered":"2024-09-21T09:48:37.860615+0000","last_clean":"2024-09-21T09:48:37.860615+0000","last_became_active":"2024-09-21T09:47:20.861988+0000","last_became_peered":"2024-09-21T09:47:20.861988+0000","last_unstale":"2024-09-21T09:48:37.860615+0000","last_undegraded":"2024-09-21T09:48:37.860615+0000","last_fullsized":"2024-09-21T09:48:37.860615+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079906+0000","last_change":"2024-09-21T09:47:24.058826+0000","last_active":"2024-09-21T09:48:38.079906+0000","last_peered":"2024-09-21T09:48:38.079906+0000","last_clean":"2024-09-21T09:48:38.079906+0000","last_became_active":"2024-09-21T09:47:24.058565+0000","last_became_peered":"2024-09-21T09:47:24.058565+0000","last_unstale":"2024-09-21T09:48:38.079906+0000","last_undegraded":"2024-09-21T09:48:38.079906+0000","last_fullsized":"2024-09-21T09:48:38.079906+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860854+0000","last_change":"2024-09-21T09:47:20.861142+0000","last_active":"2024-09-21T09:48:37.860854+0000","last_peered":"2024-09-21T09:48:37.860854+0000","last_clean":"2024-09-21T09:48:37.860854+0000","last_became_active":"2024-09-21T09:47:20.860916+0000","last_became_peered":"2024-09-21T09:47:20.860916+0000","last_unstale":"2024-09-21T09:48:37.860854+0000","last_undegraded":"2024-09-21T09:48:37.860854+0000","last_fullsized":"2024-09-21T09:48:37.860854+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080038+0000","last_change":"2024-09-21T09:47:46.481406+0000","last_active":"2024-09-21T09:48:38.080038+0000","last_peered":"2024-09-21T09:48:38.080038+0000","last_clean":"2024-09-21T09:48:38.080038+0000","last_became_active":"2024-09-21T09:47:46.481006+0000","last_became_peered":"2024-09-21T09:47:46.481006+0000","last_unstale":"2024-09-21T09:48:38.080038+0000","last_undegraded":"2024-09-21T09:48:38.080038+0000","last_fullsized":"2024-09-21T09:48:38.080038+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:18:56.780478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978483+0000","last_change":"2024-09-21T09:47:20.865234+0000","last_active":"2024-09-21T09:48:37.978483+0000","last_peered":"2024-09-21T09:48:37.978483+0000","last_clean":"2024-09-21T09:48:37.978483+0000","last_became_active":"2024-09-21T09:47:20.863790+0000","last_became_peered":"2024-09-21T09:47:20.863790+0000","last_unstale":"2024-09-21T09:48:37.978483+0000","last_undegraded":"2024-09-21T09:48:37.978483+0000","last_fullsized":"2024-09-21T09:48:37.978483+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080437+0000","last_change":"2024-09-21T09:47:24.056216+0000","last_active":"2024-09-21T09:48:38.080437+0000","last_peered":"2024-09-21T09:48:38.080437+0000","last_clean":"2024-09-21T09:48:38.080437+0000","last_became_active":"2024-09-21T09:47:24.055928+0000","last_became_peered":"2024-09-21T09:47:24.055928+0000","last_unstale":"2024-09-21T09:48:38.080437+0000","last_undegraded":"2024-09-21T09:48:38.080437+0000","last_fullsized":"2024-09-21T09:48:38.080437+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865838+0000","last_change":"2024-09-21T09:47:24.063533+0000","last_active":"2024-09-21T09:48:37.865838+0000","last_peered":"2024-09-21T09:48:37.865838+0000","last_clean":"2024-09-21T09:48:37.865838+0000","last_became_active":"2024-09-21T09:47:24.063293+0000","last_became_peered":"2024-09-21T09:47:24.063293+0000","last_unstale":"2024-09-21T09:48:37.865838+0000","last_undegraded":"2024-09-21T09:48:37.865838+0000","last_fullsized":"2024-09-21T09:48:37.865838+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978478+0000","last_change":"2024-09-21T09:47:20.864537+0000","last_active":"2024-09-21T09:48:37.978478+0000","last_peered":"2024-09-21T09:48:37.978478+0000","last_clean":"2024-09-21T09:48:37.978478+0000","last_became_active":"2024-09-21T09:47:20.864212+0000","last_became_peered":"2024-09-21T09:47:20.864212+0000","last_unstale":"2024-09-21T09:48:37.978478+0000","last_undegraded":"2024-09-21T09:48:37.978478+0000","last_fullsized":"2024-09-21T09:48:37.978478+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861706+0000","last_change":"2024-09-21T09:47:46.774197+0000","last_active":"2024-09-21T09:48:37.861706+0000","last_peered":"2024-09-21T09:48:37.861706+0000","last_clean":"2024-09-21T09:48:37.861706+0000","last_became_active":"2024-09-21T09:47:46.773846+0000","last_became_peered":"2024-09-21T09:47:46.773846+0000","last_unstale":"2024-09-21T09:48:37.861706+0000","last_undegraded":"2024-09-21T09:48:37.861706+0000","last_fullsized":"2024-09-21T09:48:37.861706+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978531+0000","last_change":"2024-09-21T09:47:20.866709+0000","last_active":"2024-09-21T09:48:37.978531+0000","last_peered":"2024-09-21T09:48:37.978531+0000","last_clean":"2024-09-21T09:48:37.978531+0000","last_became_active":"2024-09-21T09:47:20.866026+0000","last_became_peered":"2024-09-21T09:47:20.866026+0000","last_unstale":"2024-09-21T09:48:37.978531+0000","last_undegraded":"2024-09-21T09:48:37.978531+0000","last_fullsized":"2024-09-21T09:48:37.978531+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887503+0000","last_change":"2024-09-21T09:47:24.060801+0000","last_active":"2024-09-21T09:48:37.887503+0000","last_peered":"2024-09-21T09:48:37.887503+0000","last_clean":"2024-09-21T09:48:37.887503+0000","last_became_active":"2024-09-21T09:47:24.060536+0000","last_became_peered":"2024-09-21T09:47:24.060536+0000","last_unstale":"2024-09-21T09:48:37.887503+0000","last_undegraded":"2024-09-21T09:48:37.887503+0000","last_fullsized":"2024-09-21T09:48:37.887503+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080278+0000","last_change":"2024-09-21T09:47:20.864529+0000","last_active":"2024-09-21T09:48:38.080278+0000","last_peered":"2024-09-21T09:48:38.080278+0000","last_clean":"2024-09-21T09:48:38.080278+0000","last_became_active":"2024-09-21T09:47:20.864185+0000","last_became_peered":"2024-09-21T09:47:20.864185+0000","last_unstale":"2024-09-21T09:48:38.080278+0000","last_undegraded":"2024-09-21T09:48:38.080278+0000","last_fullsized":"2024-09-21T09:48:38.080278+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080170+0000","last_change":"2024-09-21T09:47:24.065677+0000","last_active":"2024-09-21T09:48:38.080170+0000","last_peered":"2024-09-21T09:48:38.080170+0000","last_clean":"2024-09-21T09:48:38.080170+0000","last_became_active":"2024-09-21T09:47:24.065373+0000","last_became_peered":"2024-09-21T09:47:24.065373+0000","last_unstale":"2024-09-21T09:48:38.080170+0000","last_undegraded":"2024-09-21T09:48:38.080170+0000","last_fullsized":"2024-09-21T09:48:38.080170+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":28,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887795+0000","last_change":"2024-09-21T09:47:20.865947+0000","last_active":"2024-09-21T09:48:37.887795+0000","last_peered":"2024-09-21T09:48:37.887795+0000","last_clean":"2024-09-21T09:48:37.887795+0000","last_became_active":"2024-09-21T09:47:20.865683+0000","last_became_peered":"2024-09-21T09:47:20.865683+0000","last_unstale":"2024-09-21T09:48:37.887795+0000","last_undegraded":"2024-09-21T09:48:37.887795+0000","last_fullsized":"2024-09-21T09:48:37.887795+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430243,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":900620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.50600000000000001,"15min":0.504},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.71199999999999997,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.46000000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.433},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.52400000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52600000000000002,"15min":0.52200000000000002},"min":{"1min":0.40300000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.42099999999999999,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.66200000000000003,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.56200000000000006}]},{"osd":4,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.502,"15min":0.51300000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.68000000000000005,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.57999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.626}]},{"osd":5,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.505,"15min":0.51700000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73199999999999998,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.55600000000000005,"15min":0.56399999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.57399999999999995,"15min":0.58599999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.71499999999999997,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.56699999999999995,"15min":0.58099999999999996},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.67500000000000004,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.65400000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626471,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1286144,"data_stored":901811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.47499999999999998,"15min":0.47099999999999997},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.67700000000000005,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.433},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.47199999999999998,"15min":0.46300000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.69199999999999995,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.65600000000000003,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.52800000000000002,"15min":0.52300000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71199999999999997,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.49199999999999999,"15min":0.48999999999999999},"min":{"1min":0.376,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.69999999999999996,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.61099999999999999,"5min":0.752,"15min":0.752},"last":0.47099999999999997}]},{"osd":3,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.50800000000000001,"15min":0.504},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66600000000000004,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.50900000000000001,"15min":0.505},"min":{"1min":0.40400000000000003,"5min":0.373,"15min":0.373},"max":{"1min":0.68799999999999994,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.41599999999999998,"15min":0.41299999999999998},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.66300000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.432,"15min":0.42899999999999999},"min":{"1min":0.30399999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.59699999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.42699999999999999,"15min":0.42199999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.63300000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.45700000000000002,"15min":0.45300000000000001},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65200000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.52100000000000002,"15min":0.52600000000000002},"min":{"1min":0.36099999999999999,"5min":0.307,"15min":0.307},"max":{"1min":0.622,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.50900000000000001,"15min":0.51400000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70399999999999996,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.61799999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822700,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298044,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447108,"statfs":{"total":95995035648,"available":95689838592,"internally_reserved":0,"allocated":704512,"data_stored":311796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48899999999999999,"15min":0.48299999999999998},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.59299999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.51400000000000001,"15min":0.51100000000000001},"min":{"1min":0.36399999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.65200000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34200000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69199999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.626},{"interface":"front","average":{"1min":0.503,"5min":0.502,"15min":0.498},"min":{"1min":0.32000000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72099999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.59099999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.52300000000000002,"15min":0.51100000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.38200000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65700000000000003,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.69799999999999995}]},{"osd":3,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52600000000000002,"15min":0.52700000000000002},"min":{"1min":0.40899999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52400000000000002,"15min":0.51400000000000001},"min":{"1min":0.36799999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73699999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.38400000000000001,"15min":0.36399999999999999},"min":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.36599999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.59899999999999998,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.59399999999999997,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.5,"15min":0.49199999999999999},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.53200000000000003,"15min":0.54000000000000004},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.70099999999999996,"5min":0.755,"15min":0.755},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.52600000000000002,"15min":0.53300000000000003},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.71299999999999997,"5min":0.752,"15min":0.752},"last":0.48699999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018929,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":310375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:48:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55700000000000005,"15min":0.57099999999999995},"min":{"1min":0.42899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.64500000000000002,"5min":0.748,"15min":0.748},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.55200000000000005,"15min":0.55000000000000004},"min":{"1min":0.40400000000000003,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.63400000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:48:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.41999999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53600000000000003,"15min":0.53100000000000003},"min":{"1min":0.36599999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.61399999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:48:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54500000000000004,"15min":0.55000000000000004},"min":{"1min":0.39200000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.53100000000000003,"15min":0.52000000000000002},"min":{"1min":0.41399999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:48:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.41999999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.55900000000000005,"15min":0.55400000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Sep 21 09:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.378,"15min":0.36199999999999999},"min":{"1min":0.25700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.60399999999999998,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.436},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.40799999999999997,"15min":0.39100000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.52900000000000003,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.51100000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.42599999999999999,"15min":0.42399999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47099999999999997,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66200000000000003,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 21 09:48:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.47499999999999998,"15min":0.48599999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.55500000000000005,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.53700000000000003,"15min":0.55100000000000005},"min":{"1min":0.32400000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.65300000000000002,"5min":0.749,"15min":0.749},"last":0.56899999999999995}]}]},{"osd":3,"up_from":25,"seq":107374182454,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298044,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447108,"statfs":{"total":95995035648,"available":95689838592,"internally_reserved":0,"allocated":704512,"data_stored":311791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.36899999999999999,"15min":0.39100000000000001},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":0.44500000000000001,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.38800000000000001,"15min":0.40799999999999997},"min":{"1min":0.22500000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.55300000000000005,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.40699999999999997,"15min":0.437},"min":{"1min":0.23400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.44,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.315},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.39900000000000002,"15min":0.42099999999999999},"min":{"1min":0.255,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.50700000000000001,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.42799999999999999,"15min":0.44800000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.50900000000000001,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.41899999999999998,"15min":0.439},"min":{"1min":0.25,"5min":0.249,"15min":0.249},"max":{"1min":0.46400000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.37}]},{"osd":4,"last update":"Sat Sep 21 09:48:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58099999999999996},"min":{"1min":0.40500000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86199999999999999,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.71899999999999997,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.85499999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:49:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.623,"15min":0.65300000000000002},"min":{"1min":0.40400000000000003,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71899999999999997,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.60499999999999998,"15min":0.621},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.66800000000000004,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.93200000000000005}]},{"osd":6,"last update":"Sat Sep 21 09:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.60099999999999998,"15min":0.60299999999999998},"min":{"1min":0.42599999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.70199999999999996,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.625,"5min":0.64500000000000002,"15min":0.64800000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.86499999999999999,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.81699999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.61299999999999999,"15min":0.622},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.71499999999999997,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.61699999999999999,"15min":0.627},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.72599999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.75800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411386,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":672,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":688128,"data_stored":309660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.379,"15min":0.35999999999999999},"min":{"1min":0.26700000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.41199999999999998,"15min":0.38900000000000001},"min":{"1min":0.28999999999999998,"5min":0.217,"15min":0.217},"max":{"1min":1.843,"5min":1.843,"15min":1.843},"last":0.27000000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.42299999999999999,"15min":0.41299999999999998},"min":{"1min":0.33100000000000002,"5min":0.254,"15min":0.254},"max":{"1min":1.9159999999999999,"5min":1.9159999999999999,"15min":1.9159999999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.45000000000000001,"15min":0.441},"min":{"1min":0.28699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.052,"5min":2.052,"15min":2.052},"last":0.37}]},{"osd":3,"last update":"Sat Sep 21 09:49:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.46000000000000002,"15min":0.47199999999999998},"min":{"1min":0.28299999999999997,"5min":0.248,"15min":0.248},"max":{"1min":2.242,"5min":2.242,"15min":2.242},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.45400000000000001,"15min":0.46100000000000002},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.22},"max":{"1min":2.0030000000000001,"5min":2.0030000000000001,"15min":2.0030000000000001},"last":0.308}]},{"osd":4,"last update":"Sat Sep 21 09:48:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58399999999999996,"15min":0.57899999999999996},"min":{"1min":0.45700000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.63},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.236,"5min":1.236,"15min":1.236},"last":0.67100000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.58999999999999997,"15min":0.57799999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.60399999999999998,"15min":0.59599999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:49:34 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.59199999999999997,"15min":0.57699999999999996},"min":{"1min":0.42899999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.61899999999999999,"15min":0.60099999999999998},"min":{"1min":0.49199999999999999,"5min":0.435,"15min":0.435},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.624,"15min":0.61799999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.622,"15min":0.61199999999999999},"min":{"1min":0.45700000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.61499999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574911,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":700416,"data_stored":309839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:48:43 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.36799999999999999,"15min":0.33000000000000002},"min":{"1min":0.223,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.36699999999999999,"15min":0.33100000000000002},"min":{"1min":0.26100000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.377},"min":{"1min":0.252,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.312},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41399999999999998,"15min":0.40899999999999997},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:49:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.441,"15min":0.44800000000000001},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.47399999999999998,"15min":0.47799999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:48:43 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58399999999999996,"15min":0.58599999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42199999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.76900000000000002,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.58499999999999996,"15min":0.57999999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 21 09:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60599999999999998,"15min":0.60399999999999998},"min":{"1min":0.44900000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.625,"15min":0.623},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.52300000000000002}]},{"osd":7,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.60699999999999998,"15min":0.60399999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.623},{"interface":"front","average":{"1min":0.63,"5min":0.622,"15min":0.62},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.63400000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738436,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1294336,"data_stored":900429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:48:46 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.35599999999999998,"15min":0.32200000000000001},"min":{"1min":0.214,"5min":0.188,"15min":0.188},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.35299999999999998},"min":{"1min":0.26400000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.60999999999999999,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.29999999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:49:03 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":0.58399999999999996,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.39600000000000002,"15min":0.38600000000000001},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.57999999999999996,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.378}]},{"osd":3,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.434,"15min":0.439},"min":{"1min":0.309,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.56399999999999995,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.25800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.315}]},{"osd":4,"last update":"Sat Sep 21 09:48:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56299999999999994,"15min":0.55300000000000005},"min":{"1min":0.35499999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.90000000000000002,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.57399999999999995,"15min":0.56799999999999995},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80100000000000005,"5min":0.871,"15min":0.871},"last":0.52400000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:49:10 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.59599999999999997,"15min":0.61199999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68400000000000005,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.59299999999999997,"15min":0.58899999999999997},"min":{"1min":0.41399999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.89200000000000002,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:48:37 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.67900000000000005,"15min":0.68700000000000006},"min":{"1min":0.42199999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.83199999999999996,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.65700000000000003,"15min":0.66800000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.80000000000000004,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:48:57 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.39600000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.88100000000000001,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.629,"15min":0.64300000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.72699999999999998,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.61599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:49:37.258 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T09:49:37.891 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:37 smithi033 ceph-mon[39045]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:49:37.892 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:37 smithi033 ceph-mon[27969]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:49:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:37 smithi105 ceph-mon[31286]: from='client.14736 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:49:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:38 smithi033 ceph-mon[39045]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:38.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:38 smithi033 ceph-mon[27969]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:38 smithi105 ceph-mon[31286]: pgmap v44: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:39.322 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:39.449 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:39 smithi033 systemd[1]: Stopping Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:49:39.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:39 smithi033 podman[79869]: 2024-09-21 09:49:39.820758714 +0000 UTC m=+0.504697614 container died 86c7c64ce06dc3776868e22301c0109dff71b8276c955fe113b0c8af8a0aa6be (image=quay.ceph.io/ceph-ci/ceph:quincy, name=dazzling_hopper, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-21T09:49:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:40 smithi105 ceph-mon[31286]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:40.783 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:40 smithi033 ceph-mon[39045]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:40.783 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:40 smithi033 ceph-mon[27969]: pgmap v45: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:41.181 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:49:41.182 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:40 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[74479]: ts=2024-09-21T09:49:40.781Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:49:41.182 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:41 smithi033 podman[79980]: 2024-09-21 09:49:41.031409844 +0000 UTC m=+0.386525674 container died a9a313d546e48b8d17e21b93e5903b5ef80384060a485dac7904ac47f1d17263 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:41.185 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-21T09:49:41.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:41 smithi033 podman[79980]: 2024-09-21 09:49:41.388944568 +0000 UTC m=+0.744060401 container cleanup a9a313d546e48b8d17e21b93e5903b5ef80384060a485dac7904ac47f1d17263 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:41.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:41 smithi033 bash[79980]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:49:41.824 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:41 smithi033 podman[80015]: 2024-09-21 09:49:41.649350029 +0000 UTC m=+0.843309190 container remove a9a313d546e48b8d17e21b93e5903b5ef80384060a485dac7904ac47f1d17263 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:41.824 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:41 smithi033 podman[80015]: 2024-09-21 09:49:41.725485135 +0000 UTC m=+0.919444295 volume remove 4f8fb96fa6ffd3735194341323fb31eee28a39eff30621dc2b220ea441ec505a 2024-09-21T09:49:42.379 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-09-21T09:49:39.299015+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":593515,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":163,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124,"ondisk_log_size":124,"up":315,"acting":315,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386052,"kb_used_data":7204,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575164,"statfs":{"total":767960285184,"available":765516967936,"internally_reserved":0,"allocated":7376896,"data_stored":4256321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978641+0000","last_change":"2024-09-21T09:47:24.066961+0000","last_active":"2024-09-21T09:48:37.978641+0000","last_peered":"2024-09-21T09:48:37.978641+0000","last_clean":"2024-09-21T09:48:37.978641+0000","last_became_active":"2024-09-21T09:47:24.066672+0000","last_became_peered":"2024-09-21T09:47:24.066672+0000","last_unstale":"2024-09-21T09:48:37.978641+0000","last_undegraded":"2024-09-21T09:48:37.978641+0000","last_fullsized":"2024-09-21T09:48:37.978641+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861122+0000","last_change":"2024-09-21T09:47:24.059165+0000","last_active":"2024-09-21T09:48:37.861122+0000","last_peered":"2024-09-21T09:48:37.861122+0000","last_clean":"2024-09-21T09:48:37.861122+0000","last_became_active":"2024-09-21T09:47:24.058702+0000","last_became_peered":"2024-09-21T09:47:24.058702+0000","last_unstale":"2024-09-21T09:48:37.861122+0000","last_undegraded":"2024-09-21T09:48:37.861122+0000","last_fullsized":"2024-09-21T09:48:37.861122+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887263+0000","last_change":"2024-09-21T09:47:24.060359+0000","last_active":"2024-09-21T09:48:37.887263+0000","last_peered":"2024-09-21T09:48:37.887263+0000","last_clean":"2024-09-21T09:48:37.887263+0000","last_became_active":"2024-09-21T09:47:24.060088+0000","last_became_peered":"2024-09-21T09:47:24.060088+0000","last_unstale":"2024-09-21T09:48:37.887263+0000","last_undegraded":"2024-09-21T09:48:37.887263+0000","last_fullsized":"2024-09-21T09:48:37.887263+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865595+0000","last_change":"2024-09-21T09:47:24.065047+0000","last_active":"2024-09-21T09:48:37.865595+0000","last_peered":"2024-09-21T09:48:37.865595+0000","last_clean":"2024-09-21T09:48:37.865595+0000","last_became_active":"2024-09-21T09:47:24.064743+0000","last_became_peered":"2024-09-21T09:47:24.064743+0000","last_unstale":"2024-09-21T09:48:37.865595+0000","last_undegraded":"2024-09-21T09:48:37.865595+0000","last_fullsized":"2024-09-21T09:48:37.865595+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978224+0000","last_change":"2024-09-21T09:47:24.069692+0000","last_active":"2024-09-21T09:48:37.978224+0000","last_peered":"2024-09-21T09:48:37.978224+0000","last_clean":"2024-09-21T09:48:37.978224+0000","last_became_active":"2024-09-21T09:47:24.069090+0000","last_became_peered":"2024-09-21T09:47:24.069090+0000","last_unstale":"2024-09-21T09:48:37.978224+0000","last_undegraded":"2024-09-21T09:48:37.978224+0000","last_fullsized":"2024-09-21T09:48:37.978224+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865499+0000","last_change":"2024-09-21T09:47:24.053540+0000","last_active":"2024-09-21T09:48:37.865499+0000","last_peered":"2024-09-21T09:48:37.865499+0000","last_clean":"2024-09-21T09:48:37.865499+0000","last_became_active":"2024-09-21T09:47:24.053163+0000","last_became_peered":"2024-09-21T09:47:24.053163+0000","last_unstale":"2024-09-21T09:48:37.865499+0000","last_undegraded":"2024-09-21T09:48:37.865499+0000","last_fullsized":"2024-09-21T09:48:37.865499+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978070+0000","last_change":"2024-09-21T09:47:46.774760+0000","last_active":"2024-09-21T09:48:37.978070+0000","last_peered":"2024-09-21T09:48:37.978070+0000","last_clean":"2024-09-21T09:48:37.978070+0000","last_became_active":"2024-09-21T09:47:46.774445+0000","last_became_peered":"2024-09-21T09:47:46.774445+0000","last_unstale":"2024-09-21T09:48:37.978070+0000","last_undegraded":"2024-09-21T09:48:37.978070+0000","last_fullsized":"2024-09-21T09:48:37.978070+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079815+0000","last_change":"2024-09-21T09:47:24.064091+0000","last_active":"2024-09-21T09:48:38.079815+0000","last_peered":"2024-09-21T09:48:38.079815+0000","last_clean":"2024-09-21T09:48:38.079815+0000","last_became_active":"2024-09-21T09:47:24.063698+0000","last_became_peered":"2024-09-21T09:47:24.063698+0000","last_unstale":"2024-09-21T09:48:38.079815+0000","last_undegraded":"2024-09-21T09:48:38.079815+0000","last_fullsized":"2024-09-21T09:48:38.079815+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978791+0000","last_change":"2024-09-21T09:47:24.071747+0000","last_active":"2024-09-21T09:48:37.978791+0000","last_peered":"2024-09-21T09:48:37.978791+0000","last_clean":"2024-09-21T09:48:37.978791+0000","last_became_active":"2024-09-21T09:47:24.071475+0000","last_became_peered":"2024-09-21T09:47:24.071475+0000","last_unstale":"2024-09-21T09:48:37.978791+0000","last_undegraded":"2024-09-21T09:48:37.978791+0000","last_fullsized":"2024-09-21T09:48:37.978791+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080913+0000","last_change":"2024-09-21T09:47:24.058216+0000","last_active":"2024-09-21T09:48:38.080913+0000","last_peered":"2024-09-21T09:48:38.080913+0000","last_clean":"2024-09-21T09:48:38.080913+0000","last_became_active":"2024-09-21T09:47:24.057940+0000","last_became_peered":"2024-09-21T09:47:24.057940+0000","last_unstale":"2024-09-21T09:48:38.080913+0000","last_undegraded":"2024-09-21T09:48:38.080913+0000","last_fullsized":"2024-09-21T09:48:38.080913+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080816+0000","last_change":"2024-09-21T09:47:24.053543+0000","last_active":"2024-09-21T09:48:38.080816+0000","last_peered":"2024-09-21T09:48:38.080816+0000","last_clean":"2024-09-21T09:48:38.080816+0000","last_became_active":"2024-09-21T09:47:24.053264+0000","last_became_peered":"2024-09-21T09:47:24.053264+0000","last_unstale":"2024-09-21T09:48:38.080816+0000","last_undegraded":"2024-09-21T09:48:38.080816+0000","last_fullsized":"2024-09-21T09:48:38.080816+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861618+0000","last_change":"2024-09-21T09:47:46.850968+0000","last_active":"2024-09-21T09:48:37.861618+0000","last_peered":"2024-09-21T09:48:37.861618+0000","last_clean":"2024-09-21T09:48:37.861618+0000","last_became_active":"2024-09-21T09:47:46.850604+0000","last_became_peered":"2024-09-21T09:47:46.850604+0000","last_unstale":"2024-09-21T09:48:37.861618+0000","last_undegraded":"2024-09-21T09:48:37.861618+0000","last_fullsized":"2024-09-21T09:48:37.861618+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080096+0000","last_change":"2024-09-21T09:47:24.064334+0000","last_active":"2024-09-21T09:48:38.080096+0000","last_peered":"2024-09-21T09:48:38.080096+0000","last_clean":"2024-09-21T09:48:38.080096+0000","last_became_active":"2024-09-21T09:47:24.064063+0000","last_became_peered":"2024-09-21T09:47:24.064063+0000","last_unstale":"2024-09-21T09:48:38.080096+0000","last_undegraded":"2024-09-21T09:48:38.080096+0000","last_fullsized":"2024-09-21T09:48:38.080096+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886822+0000","last_change":"2024-09-21T09:47:24.058842+0000","last_active":"2024-09-21T09:48:37.886822+0000","last_peered":"2024-09-21T09:48:37.886822+0000","last_clean":"2024-09-21T09:48:37.886822+0000","last_became_active":"2024-09-21T09:47:24.058245+0000","last_became_peered":"2024-09-21T09:47:24.058245+0000","last_unstale":"2024-09-21T09:48:37.886822+0000","last_undegraded":"2024-09-21T09:48:37.886822+0000","last_fullsized":"2024-09-21T09:48:37.886822+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860590+0000","last_change":"2024-09-21T09:47:24.052028+0000","last_active":"2024-09-21T09:48:37.860590+0000","last_peered":"2024-09-21T09:48:37.860590+0000","last_clean":"2024-09-21T09:48:37.860590+0000","last_became_active":"2024-09-21T09:47:24.051766+0000","last_became_peered":"2024-09-21T09:47:24.051766+0000","last_unstale":"2024-09-21T09:48:37.860590+0000","last_undegraded":"2024-09-21T09:48:37.860590+0000","last_fullsized":"2024-09-21T09:48:37.860590+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865236+0000","last_change":"2024-09-21T09:47:24.060117+0000","last_active":"2024-09-21T09:48:37.865236+0000","last_peered":"2024-09-21T09:48:37.865236+0000","last_clean":"2024-09-21T09:48:37.865236+0000","last_became_active":"2024-09-21T09:47:24.059852+0000","last_became_peered":"2024-09-21T09:47:24.059852+0000","last_unstale":"2024-09-21T09:48:37.865236+0000","last_undegraded":"2024-09-21T09:48:37.865236+0000","last_fullsized":"2024-09-21T09:48:37.865236+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865999+0000","last_change":"2024-09-21T09:47:46.480400+0000","last_active":"2024-09-21T09:48:37.865999+0000","last_peered":"2024-09-21T09:48:37.865999+0000","last_clean":"2024-09-21T09:48:37.865999+0000","last_became_active":"2024-09-21T09:47:46.480103+0000","last_became_peered":"2024-09-21T09:47:46.480103+0000","last_unstale":"2024-09-21T09:48:37.865999+0000","last_undegraded":"2024-09-21T09:48:37.865999+0000","last_fullsized":"2024-09-21T09:48:37.865999+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.862210+0000","last_change":"2024-09-21T09:47:24.057147+0000","last_active":"2024-09-21T09:48:37.862210+0000","last_peered":"2024-09-21T09:48:37.862210+0000","last_clean":"2024-09-21T09:48:37.862210+0000","last_became_active":"2024-09-21T09:47:24.056798+0000","last_became_peered":"2024-09-21T09:47:24.056798+0000","last_unstale":"2024-09-21T09:48:37.862210+0000","last_undegraded":"2024-09-21T09:48:37.862210+0000","last_fullsized":"2024-09-21T09:48:37.862210+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978661+0000","last_change":"2024-09-21T09:47:24.070640+0000","last_active":"2024-09-21T09:48:37.978661+0000","last_peered":"2024-09-21T09:48:37.978661+0000","last_clean":"2024-09-21T09:48:37.978661+0000","last_became_active":"2024-09-21T09:47:24.070359+0000","last_became_peered":"2024-09-21T09:47:24.070359+0000","last_unstale":"2024-09-21T09:48:37.978661+0000","last_undegraded":"2024-09-21T09:48:37.978661+0000","last_fullsized":"2024-09-21T09:48:37.978661+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080608+0000","last_change":"2024-09-21T09:47:24.056659+0000","last_active":"2024-09-21T09:48:38.080608+0000","last_peered":"2024-09-21T09:48:38.080608+0000","last_clean":"2024-09-21T09:48:38.080608+0000","last_became_active":"2024-09-21T09:47:24.056360+0000","last_became_peered":"2024-09-21T09:47:24.056360+0000","last_unstale":"2024-09-21T09:48:38.080608+0000","last_undegraded":"2024-09-21T09:48:38.080608+0000","last_fullsized":"2024-09-21T09:48:38.080608+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886909+0000","last_change":"2024-09-21T09:47:24.065154+0000","last_active":"2024-09-21T09:48:37.886909+0000","last_peered":"2024-09-21T09:48:37.886909+0000","last_clean":"2024-09-21T09:48:37.886909+0000","last_became_active":"2024-09-21T09:47:24.064231+0000","last_became_peered":"2024-09-21T09:47:24.064231+0000","last_unstale":"2024-09-21T09:48:37.886909+0000","last_undegraded":"2024-09-21T09:48:37.886909+0000","last_fullsized":"2024-09-21T09:48:37.886909+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865558+0000","last_change":"2024-09-21T09:47:24.062716+0000","last_active":"2024-09-21T09:48:37.865558+0000","last_peered":"2024-09-21T09:48:37.865558+0000","last_clean":"2024-09-21T09:48:37.865558+0000","last_became_active":"2024-09-21T09:47:24.062437+0000","last_became_peered":"2024-09-21T09:47:24.062437+0000","last_unstale":"2024-09-21T09:48:37.865558+0000","last_undegraded":"2024-09-21T09:48:37.865558+0000","last_fullsized":"2024-09-21T09:48:37.865558+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079839+0000","last_change":"2024-09-21T09:47:24.051387+0000","last_active":"2024-09-21T09:48:38.079839+0000","last_peered":"2024-09-21T09:48:38.079839+0000","last_clean":"2024-09-21T09:48:38.079839+0000","last_became_active":"2024-09-21T09:47:24.051090+0000","last_became_peered":"2024-09-21T09:47:24.051090+0000","last_unstale":"2024-09-21T09:48:38.079839+0000","last_undegraded":"2024-09-21T09:48:38.079839+0000","last_fullsized":"2024-09-21T09:48:38.079839+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865298+0000","last_change":"2024-09-21T09:47:24.058079+0000","last_active":"2024-09-21T09:48:37.865298+0000","last_peered":"2024-09-21T09:48:37.865298+0000","last_clean":"2024-09-21T09:48:37.865298+0000","last_became_active":"2024-09-21T09:47:24.057753+0000","last_became_peered":"2024-09-21T09:47:24.057753+0000","last_unstale":"2024-09-21T09:48:37.865298+0000","last_undegraded":"2024-09-21T09:48:37.865298+0000","last_fullsized":"2024-09-21T09:48:37.865298+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080292+0000","last_change":"2024-09-21T09:47:24.057220+0000","last_active":"2024-09-21T09:48:38.080292+0000","last_peered":"2024-09-21T09:48:38.080292+0000","last_clean":"2024-09-21T09:48:38.080292+0000","last_became_active":"2024-09-21T09:47:24.056915+0000","last_became_peered":"2024-09-21T09:47:24.056915+0000","last_unstale":"2024-09-21T09:48:38.080292+0000","last_undegraded":"2024-09-21T09:48:38.080292+0000","last_fullsized":"2024-09-21T09:48:38.080292+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080917+0000","last_change":"2024-09-21T09:47:24.057310+0000","last_active":"2024-09-21T09:48:38.080917+0000","last_peered":"2024-09-21T09:48:38.080917+0000","last_clean":"2024-09-21T09:48:38.080917+0000","last_became_active":"2024-09-21T09:47:24.057047+0000","last_became_peered":"2024-09-21T09:47:24.057047+0000","last_unstale":"2024-09-21T09:48:38.080917+0000","last_undegraded":"2024-09-21T09:48:38.080917+0000","last_fullsized":"2024-09-21T09:48:38.080917+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861084+0000","last_change":"2024-09-21T09:47:24.060976+0000","last_active":"2024-09-21T09:48:37.861084+0000","last_peered":"2024-09-21T09:48:37.861084+0000","last_clean":"2024-09-21T09:48:37.861084+0000","last_became_active":"2024-09-21T09:47:24.060725+0000","last_became_peered":"2024-09-21T09:47:24.060725+0000","last_unstale":"2024-09-21T09:48:37.861084+0000","last_undegraded":"2024-09-21T09:48:37.861084+0000","last_fullsized":"2024-09-21T09:48:37.861084+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080326+0000","last_change":"2024-09-21T09:47:24.053563+0000","last_active":"2024-09-21T09:48:38.080326+0000","last_peered":"2024-09-21T09:48:38.080326+0000","last_clean":"2024-09-21T09:48:38.080326+0000","last_became_active":"2024-09-21T09:47:24.052853+0000","last_became_peered":"2024-09-21T09:47:24.052853+0000","last_unstale":"2024-09-21T09:48:38.080326+0000","last_undegraded":"2024-09-21T09:48:38.080326+0000","last_fullsized":"2024-09-21T09:48:38.080326+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886944+0000","last_change":"2024-09-21T09:47:24.068204+0000","last_active":"2024-09-21T09:48:37.886944+0000","last_peered":"2024-09-21T09:48:37.886944+0000","last_clean":"2024-09-21T09:48:37.886944+0000","last_became_active":"2024-09-21T09:47:24.067911+0000","last_became_peered":"2024-09-21T09:47:24.067911+0000","last_unstale":"2024-09-21T09:48:37.886944+0000","last_undegraded":"2024-09-21T09:48:37.886944+0000","last_fullsized":"2024-09-21T09:48:37.886944+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865668+0000","last_change":"2024-09-21T09:47:24.058882+0000","last_active":"2024-09-21T09:48:37.865668+0000","last_peered":"2024-09-21T09:48:37.865668+0000","last_clean":"2024-09-21T09:48:37.865668+0000","last_became_active":"2024-09-21T09:47:24.058587+0000","last_became_peered":"2024-09-21T09:47:24.058587+0000","last_unstale":"2024-09-21T09:48:37.865668+0000","last_undegraded":"2024-09-21T09:48:37.865668+0000","last_fullsized":"2024-09-21T09:48:37.865668+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861701+0000","last_change":"2024-09-21T09:47:24.060446+0000","last_active":"2024-09-21T09:48:37.861701+0000","last_peered":"2024-09-21T09:48:37.861701+0000","last_clean":"2024-09-21T09:48:37.861701+0000","last_became_active":"2024-09-21T09:47:24.060165+0000","last_became_peered":"2024-09-21T09:47:24.060165+0000","last_unstale":"2024-09-21T09:48:37.861701+0000","last_undegraded":"2024-09-21T09:48:37.861701+0000","last_fullsized":"2024-09-21T09:48:37.861701+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080189+0000","last_change":"2024-09-21T09:47:24.059974+0000","last_active":"2024-09-21T09:48:38.080189+0000","last_peered":"2024-09-21T09:48:38.080189+0000","last_clean":"2024-09-21T09:48:38.080189+0000","last_became_active":"2024-09-21T09:47:24.059270+0000","last_became_peered":"2024-09-21T09:47:24.059270+0000","last_unstale":"2024-09-21T09:48:38.080189+0000","last_undegraded":"2024-09-21T09:48:38.080189+0000","last_fullsized":"2024-09-21T09:48:38.080189+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.859975+0000","last_change":"2024-09-21T09:47:24.048941+0000","last_active":"2024-09-21T09:48:37.859975+0000","last_peered":"2024-09-21T09:48:37.859975+0000","last_clean":"2024-09-21T09:48:37.859975+0000","last_became_active":"2024-09-21T09:47:24.048631+0000","last_became_peered":"2024-09-21T09:47:24.048631+0000","last_unstale":"2024-09-21T09:48:37.859975+0000","last_undegraded":"2024-09-21T09:48:37.859975+0000","last_fullsized":"2024-09-21T09:48:37.859975+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080417+0000","last_change":"2024-09-21T09:47:20.866010+0000","last_active":"2024-09-21T09:48:38.080417+0000","last_peered":"2024-09-21T09:48:38.080417+0000","last_clean":"2024-09-21T09:48:38.080417+0000","last_became_active":"2024-09-21T09:47:20.865732+0000","last_became_peered":"2024-09-21T09:47:20.865732+0000","last_unstale":"2024-09-21T09:48:38.080417+0000","last_undegraded":"2024-09-21T09:48:38.080417+0000","last_fullsized":"2024-09-21T09:48:38.080417+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886995+0000","last_change":"2024-09-21T09:47:24.067813+0000","last_active":"2024-09-21T09:48:37.886995+0000","last_peered":"2024-09-21T09:48:37.886995+0000","last_clean":"2024-09-21T09:48:37.886995+0000","last_became_active":"2024-09-21T09:47:24.067412+0000","last_became_peered":"2024-09-21T09:47:24.067412+0000","last_unstale":"2024-09-21T09:48:37.886995+0000","last_undegraded":"2024-09-21T09:48:37.886995+0000","last_fullsized":"2024-09-21T09:48:37.886995+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"57'1","reported_seq":35,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080597+0000","last_change":"2024-09-21T09:47:46.481320+0000","last_active":"2024-09-21T09:48:38.080597+0000","last_peered":"2024-09-21T09:48:38.080597+0000","last_clean":"2024-09-21T09:48:38.080597+0000","last_became_active":"2024-09-21T09:47:46.480905+0000","last_became_peered":"2024-09-21T09:47:46.480905+0000","last_unstale":"2024-09-21T09:48:38.080597+0000","last_undegraded":"2024-09-21T09:48:38.080597+0000","last_fullsized":"2024-09-21T09:48:38.080597+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:58:23.831301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079961+0000","last_change":"2024-09-21T09:47:20.859829+0000","last_active":"2024-09-21T09:48:38.079961+0000","last_peered":"2024-09-21T09:48:38.079961+0000","last_clean":"2024-09-21T09:48:38.079961+0000","last_became_active":"2024-09-21T09:47:20.859507+0000","last_became_peered":"2024-09-21T09:47:20.859507+0000","last_unstale":"2024-09-21T09:48:38.079961+0000","last_undegraded":"2024-09-21T09:48:38.079961+0000","last_fullsized":"2024-09-21T09:48:38.079961+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079902+0000","last_change":"2024-09-21T09:47:24.053702+0000","last_active":"2024-09-21T09:48:38.079902+0000","last_peered":"2024-09-21T09:48:38.079902+0000","last_clean":"2024-09-21T09:48:38.079902+0000","last_became_active":"2024-09-21T09:47:24.053295+0000","last_became_peered":"2024-09-21T09:47:24.053295+0000","last_unstale":"2024-09-21T09:48:38.079902+0000","last_undegraded":"2024-09-21T09:48:38.079902+0000","last_fullsized":"2024-09-21T09:48:38.079902+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865313+0000","last_change":"2024-09-21T09:47:20.863181+0000","last_active":"2024-09-21T09:48:37.865313+0000","last_peered":"2024-09-21T09:48:37.865313+0000","last_clean":"2024-09-21T09:48:37.865313+0000","last_became_active":"2024-09-21T09:47:20.862895+0000","last_became_peered":"2024-09-21T09:47:20.862895+0000","last_unstale":"2024-09-21T09:48:37.865313+0000","last_undegraded":"2024-09-21T09:48:37.865313+0000","last_fullsized":"2024-09-21T09:48:37.865313+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860485+0000","last_change":"2024-09-21T09:47:24.051248+0000","last_active":"2024-09-21T09:48:37.860485+0000","last_peered":"2024-09-21T09:48:37.860485+0000","last_clean":"2024-09-21T09:48:37.860485+0000","last_became_active":"2024-09-21T09:47:24.050918+0000","last_became_peered":"2024-09-21T09:47:24.050918+0000","last_unstale":"2024-09-21T09:48:37.860485+0000","last_undegraded":"2024-09-21T09:48:37.860485+0000","last_fullsized":"2024-09-21T09:48:37.860485+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080973+0000","last_change":"2024-09-21T09:47:20.865532+0000","last_active":"2024-09-21T09:48:38.080973+0000","last_peered":"2024-09-21T09:48:38.080973+0000","last_clean":"2024-09-21T09:48:38.080973+0000","last_became_active":"2024-09-21T09:47:20.865103+0000","last_became_peered":"2024-09-21T09:47:20.865103+0000","last_unstale":"2024-09-21T09:48:38.080973+0000","last_undegraded":"2024-09-21T09:48:38.080973+0000","last_fullsized":"2024-09-21T09:48:38.080973+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865804+0000","last_change":"2024-09-21T09:47:24.067330+0000","last_active":"2024-09-21T09:48:37.865804+0000","last_peered":"2024-09-21T09:48:37.865804+0000","last_clean":"2024-09-21T09:48:37.865804+0000","last_became_active":"2024-09-21T09:47:24.066939+0000","last_became_peered":"2024-09-21T09:47:24.066939+0000","last_unstale":"2024-09-21T09:48:37.865804+0000","last_undegraded":"2024-09-21T09:48:37.865804+0000","last_fullsized":"2024-09-21T09:48:37.865804+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865951+0000","last_change":"2024-09-21T09:47:20.860605+0000","last_active":"2024-09-21T09:48:37.865951+0000","last_peered":"2024-09-21T09:48:37.865951+0000","last_clean":"2024-09-21T09:48:37.865951+0000","last_became_active":"2024-09-21T09:47:20.860288+0000","last_became_peered":"2024-09-21T09:47:20.860288+0000","last_unstale":"2024-09-21T09:48:37.865951+0000","last_undegraded":"2024-09-21T09:48:37.865951+0000","last_fullsized":"2024-09-21T09:48:37.865951+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887587+0000","last_change":"2024-09-21T09:47:24.057860+0000","last_active":"2024-09-21T09:48:37.887587+0000","last_peered":"2024-09-21T09:48:37.887587+0000","last_clean":"2024-09-21T09:48:37.887587+0000","last_became_active":"2024-09-21T09:47:24.057594+0000","last_became_peered":"2024-09-21T09:47:24.057594+0000","last_unstale":"2024-09-21T09:48:37.887587+0000","last_undegraded":"2024-09-21T09:48:37.887587+0000","last_fullsized":"2024-09-21T09:48:37.887587+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":27,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860960+0000","last_change":"2024-09-21T09:47:20.857026+0000","last_active":"2024-09-21T09:48:37.860960+0000","last_peered":"2024-09-21T09:48:37.860960+0000","last_clean":"2024-09-21T09:48:37.860960+0000","last_became_active":"2024-09-21T09:47:20.856719+0000","last_became_peered":"2024-09-21T09:47:20.856719+0000","last_unstale":"2024-09-21T09:48:37.860960+0000","last_undegraded":"2024-09-21T09:48:37.860960+0000","last_fullsized":"2024-09-21T09:48:37.860960+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860234+0000","last_change":"2024-09-21T09:47:24.059537+0000","last_active":"2024-09-21T09:48:37.860234+0000","last_peered":"2024-09-21T09:48:37.860234+0000","last_clean":"2024-09-21T09:48:37.860234+0000","last_became_active":"2024-09-21T09:47:24.059264+0000","last_became_peered":"2024-09-21T09:47:24.059264+0000","last_unstale":"2024-09-21T09:48:37.860234+0000","last_undegraded":"2024-09-21T09:48:37.860234+0000","last_fullsized":"2024-09-21T09:48:37.860234+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"60'9","reported_seq":33,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080662+0000","last_change":"2024-09-21T09:47:20.864446+0000","last_active":"2024-09-21T09:48:38.080662+0000","last_peered":"2024-09-21T09:48:38.080662+0000","last_clean":"2024-09-21T09:48:38.080662+0000","last_became_active":"2024-09-21T09:47:20.864160+0000","last_became_peered":"2024-09-21T09:47:20.864160+0000","last_unstale":"2024-09-21T09:48:38.080662+0000","last_undegraded":"2024-09-21T09:48:38.080662+0000","last_fullsized":"2024-09-21T09:48:38.080662+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":9,"log_dups_size":0,"ondisk_log_size":9,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2342,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978421+0000","last_change":"2024-09-21T09:47:24.070010+0000","last_active":"2024-09-21T09:48:37.978421+0000","last_peered":"2024-09-21T09:48:37.978421+0000","last_clean":"2024-09-21T09:48:37.978421+0000","last_became_active":"2024-09-21T09:47:24.069698+0000","last_became_peered":"2024-09-21T09:47:24.069698+0000","last_unstale":"2024-09-21T09:48:37.978421+0000","last_undegraded":"2024-09-21T09:48:37.978421+0000","last_fullsized":"2024-09-21T09:48:37.978421+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887044+0000","last_change":"2024-09-21T09:47:20.858242+0000","last_active":"2024-09-21T09:48:37.887044+0000","last_peered":"2024-09-21T09:48:37.887044+0000","last_clean":"2024-09-21T09:48:37.887044+0000","last_became_active":"2024-09-21T09:47:20.857951+0000","last_became_peered":"2024-09-21T09:47:20.857951+0000","last_unstale":"2024-09-21T09:48:37.887044+0000","last_undegraded":"2024-09-21T09:48:37.887044+0000","last_fullsized":"2024-09-21T09:48:37.887044+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886995+0000","last_change":"2024-09-21T09:47:24.063666+0000","last_active":"2024-09-21T09:48:37.886995+0000","last_peered":"2024-09-21T09:48:37.886995+0000","last_clean":"2024-09-21T09:48:37.886995+0000","last_became_active":"2024-09-21T09:47:24.063116+0000","last_became_peered":"2024-09-21T09:47:24.063116+0000","last_unstale":"2024-09-21T09:48:37.886995+0000","last_undegraded":"2024-09-21T09:48:37.886995+0000","last_fullsized":"2024-09-21T09:48:37.886995+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865774+0000","last_change":"2024-09-21T09:47:20.867134+0000","last_active":"2024-09-21T09:48:37.865774+0000","last_peered":"2024-09-21T09:48:37.865774+0000","last_clean":"2024-09-21T09:48:37.865774+0000","last_became_active":"2024-09-21T09:47:20.866875+0000","last_became_peered":"2024-09-21T09:47:20.866875+0000","last_unstale":"2024-09-21T09:48:37.865774+0000","last_undegraded":"2024-09-21T09:48:37.865774+0000","last_fullsized":"2024-09-21T09:48:37.865774+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978277+0000","last_change":"2024-09-21T09:47:24.069831+0000","last_active":"2024-09-21T09:48:37.978277+0000","last_peered":"2024-09-21T09:48:37.978277+0000","last_clean":"2024-09-21T09:48:37.978277+0000","last_became_active":"2024-09-21T09:47:24.069569+0000","last_became_peered":"2024-09-21T09:47:24.069569+0000","last_unstale":"2024-09-21T09:48:37.978277+0000","last_undegraded":"2024-09-21T09:48:37.978277+0000","last_fullsized":"2024-09-21T09:48:37.978277+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080355+0000","last_change":"2024-09-21T09:47:20.862992+0000","last_active":"2024-09-21T09:48:38.080355+0000","last_peered":"2024-09-21T09:48:38.080355+0000","last_clean":"2024-09-21T09:48:38.080355+0000","last_became_active":"2024-09-21T09:47:20.862582+0000","last_became_peered":"2024-09-21T09:47:20.862582+0000","last_unstale":"2024-09-21T09:48:38.080355+0000","last_undegraded":"2024-09-21T09:48:38.080355+0000","last_fullsized":"2024-09-21T09:48:38.080355+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080230+0000","last_change":"2024-09-21T09:47:24.058311+0000","last_active":"2024-09-21T09:48:38.080230+0000","last_peered":"2024-09-21T09:48:38.080230+0000","last_clean":"2024-09-21T09:48:38.080230+0000","last_became_active":"2024-09-21T09:47:24.057961+0000","last_became_peered":"2024-09-21T09:47:24.057961+0000","last_unstale":"2024-09-21T09:48:38.080230+0000","last_undegraded":"2024-09-21T09:48:38.080230+0000","last_fullsized":"2024-09-21T09:48:38.080230+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":32,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860182+0000","last_change":"2024-09-21T09:47:20.856509+0000","last_active":"2024-09-21T09:48:37.860182+0000","last_peered":"2024-09-21T09:48:37.860182+0000","last_clean":"2024-09-21T09:48:37.860182+0000","last_became_active":"2024-09-21T09:47:20.856229+0000","last_became_peered":"2024-09-21T09:47:20.856229+0000","last_unstale":"2024-09-21T09:48:37.860182+0000","last_undegraded":"2024-09-21T09:48:37.860182+0000","last_fullsized":"2024-09-21T09:48:37.860182+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978178+0000","last_change":"2024-09-21T09:47:24.069301+0000","last_active":"2024-09-21T09:48:37.978178+0000","last_peered":"2024-09-21T09:48:37.978178+0000","last_clean":"2024-09-21T09:48:37.978178+0000","last_became_active":"2024-09-21T09:47:24.068917+0000","last_became_peered":"2024-09-21T09:47:24.068917+0000","last_unstale":"2024-09-21T09:48:37.978178+0000","last_undegraded":"2024-09-21T09:48:37.978178+0000","last_fullsized":"2024-09-21T09:48:37.978178+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.f","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860611+0000","last_change":"2024-09-21T09:47:20.862118+0000","last_active":"2024-09-21T09:48:37.860611+0000","last_peered":"2024-09-21T09:48:37.860611+0000","last_clean":"2024-09-21T09:48:37.860611+0000","last_became_active":"2024-09-21T09:47:20.861815+0000","last_became_peered":"2024-09-21T09:47:20.861815+0000","last_unstale":"2024-09-21T09:48:37.860611+0000","last_undegraded":"2024-09-21T09:48:37.860611+0000","last_fullsized":"2024-09-21T09:48:37.860611+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978644+0000","last_change":"2024-09-21T09:47:24.069502+0000","last_active":"2024-09-21T09:48:37.978644+0000","last_peered":"2024-09-21T09:48:37.978644+0000","last_clean":"2024-09-21T09:48:37.978644+0000","last_became_active":"2024-09-21T09:47:24.069235+0000","last_became_peered":"2024-09-21T09:47:24.069235+0000","last_unstale":"2024-09-21T09:48:37.978644+0000","last_undegraded":"2024-09-21T09:48:37.978644+0000","last_fullsized":"2024-09-21T09:48:37.978644+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.e","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.862390+0000","last_change":"2024-09-21T09:47:20.862503+0000","last_active":"2024-09-21T09:48:37.862390+0000","last_peered":"2024-09-21T09:48:37.862390+0000","last_clean":"2024-09-21T09:48:37.862390+0000","last_became_active":"2024-09-21T09:47:20.862237+0000","last_became_peered":"2024-09-21T09:47:20.862237+0000","last_unstale":"2024-09-21T09:48:37.862390+0000","last_undegraded":"2024-09-21T09:48:37.862390+0000","last_fullsized":"2024-09-21T09:48:37.862390+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978686+0000","last_change":"2024-09-21T09:47:24.069273+0000","last_active":"2024-09-21T09:48:37.978686+0000","last_peered":"2024-09-21T09:48:37.978686+0000","last_clean":"2024-09-21T09:48:37.978686+0000","last_became_active":"2024-09-21T09:47:24.068990+0000","last_became_peered":"2024-09-21T09:47:24.068990+0000","last_unstale":"2024-09-21T09:48:37.978686+0000","last_undegraded":"2024-09-21T09:48:37.978686+0000","last_fullsized":"2024-09-21T09:48:37.978686+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.d","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861047+0000","last_change":"2024-09-21T09:47:20.858095+0000","last_active":"2024-09-21T09:48:37.861047+0000","last_peered":"2024-09-21T09:48:37.861047+0000","last_clean":"2024-09-21T09:48:37.861047+0000","last_became_active":"2024-09-21T09:47:20.857760+0000","last_became_peered":"2024-09-21T09:47:20.857760+0000","last_unstale":"2024-09-21T09:48:37.861047+0000","last_undegraded":"2024-09-21T09:48:37.861047+0000","last_fullsized":"2024-09-21T09:48:37.861047+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865642+0000","last_change":"2024-09-21T09:47:24.063342+0000","last_active":"2024-09-21T09:48:37.865642+0000","last_peered":"2024-09-21T09:48:37.865642+0000","last_clean":"2024-09-21T09:48:37.865642+0000","last_became_active":"2024-09-21T09:47:24.063070+0000","last_became_peered":"2024-09-21T09:47:24.063070+0000","last_unstale":"2024-09-21T09:48:37.865642+0000","last_undegraded":"2024-09-21T09:48:37.865642+0000","last_fullsized":"2024-09-21T09:48:37.865642+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.c","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861853+0000","last_change":"2024-09-21T09:47:20.859702+0000","last_active":"2024-09-21T09:48:37.861853+0000","last_peered":"2024-09-21T09:48:37.861853+0000","last_clean":"2024-09-21T09:48:37.861853+0000","last_became_active":"2024-09-21T09:47:20.859443+0000","last_became_peered":"2024-09-21T09:47:20.859443+0000","last_unstale":"2024-09-21T09:48:37.861853+0000","last_undegraded":"2024-09-21T09:48:37.861853+0000","last_fullsized":"2024-09-21T09:48:37.861853+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978522+0000","last_change":"2024-09-21T09:47:24.070338+0000","last_active":"2024-09-21T09:48:37.978522+0000","last_peered":"2024-09-21T09:48:37.978522+0000","last_clean":"2024-09-21T09:48:37.978522+0000","last_became_active":"2024-09-21T09:47:24.070074+0000","last_became_peered":"2024-09-21T09:47:24.070074+0000","last_unstale":"2024-09-21T09:48:37.978522+0000","last_undegraded":"2024-09-21T09:48:37.978522+0000","last_fullsized":"2024-09-21T09:48:37.978522+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.656763+0000","last_change":"2024-09-21T09:48:38.081026+0000","last_active":"2024-09-21T09:48:41.656763+0000","last_peered":"2024-09-21T09:48:41.656763+0000","last_clean":"2024-09-21T09:48:41.656763+0000","last_became_active":"2024-09-21T09:48:33.296475+0000","last_became_peered":"2024-09-21T09:48:33.296475+0000","last_unstale":"2024-09-21T09:48:41.656763+0000","last_undegraded":"2024-09-21T09:48:41.656763+0000","last_fullsized":"2024-09-21T09:48:41.656763+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:24:15.020915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046213800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"60'79","reported_seq":58,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978200+0000","last_change":"2024-09-21T09:46:57.426595+0000","last_active":"2024-09-21T09:48:37.978200+0000","last_peered":"2024-09-21T09:48:37.978200+0000","last_clean":"2024-09-21T09:48:37.978200+0000","last_became_active":"2024-09-21T09:46:57.418997+0000","last_became_peered":"2024-09-21T09:46:57.418997+0000","last_unstale":"2024-09-21T09:48:37.978200+0000","last_undegraded":"2024-09-21T09:48:37.978200+0000","last_fullsized":"2024-09-21T09:48:37.978200+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_clean_scrub_stamp":"2024-09-21T09:44:57.819716+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865760+0000","last_change":"2024-09-21T09:47:20.866500+0000","last_active":"2024-09-21T09:48:37.865760+0000","last_peered":"2024-09-21T09:48:37.865760+0000","last_clean":"2024-09-21T09:48:37.865760+0000","last_became_active":"2024-09-21T09:47:20.866224+0000","last_became_peered":"2024-09-21T09:47:20.866224+0000","last_unstale":"2024-09-21T09:48:37.865760+0000","last_undegraded":"2024-09-21T09:48:37.865760+0000","last_fullsized":"2024-09-21T09:48:37.865760+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079954+0000","last_change":"2024-09-21T09:47:24.061053+0000","last_active":"2024-09-21T09:48:38.079954+0000","last_peered":"2024-09-21T09:48:38.079954+0000","last_clean":"2024-09-21T09:48:38.079954+0000","last_became_active":"2024-09-21T09:47:24.060632+0000","last_became_peered":"2024-09-21T09:47:24.060632+0000","last_unstale":"2024-09-21T09:48:38.079954+0000","last_undegraded":"2024-09-21T09:48:38.079954+0000","last_fullsized":"2024-09-21T09:48:38.079954+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.509038+0000","last_change":"2024-09-21T09:48:37.887537+0000","last_active":"2024-09-21T09:48:41.509038+0000","last_peered":"2024-09-21T09:48:41.509038+0000","last_clean":"2024-09-21T09:48:41.509038+0000","last_became_active":"2024-09-21T09:48:33.298887+0000","last_became_peered":"2024-09-21T09:48:33.298887+0000","last_unstale":"2024-09-21T09:48:41.509038+0000","last_undegraded":"2024-09-21T09:48:41.509038+0000","last_fullsized":"2024-09-21T09:48:41.509038+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:51:16.078381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027730799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978152+0000","last_change":"2024-09-21T09:47:20.869192+0000","last_active":"2024-09-21T09:48:37.978152+0000","last_peered":"2024-09-21T09:48:37.978152+0000","last_clean":"2024-09-21T09:48:37.978152+0000","last_became_active":"2024-09-21T09:47:20.868800+0000","last_became_peered":"2024-09-21T09:47:20.868800+0000","last_unstale":"2024-09-21T09:48:37.978152+0000","last_undegraded":"2024-09-21T09:48:37.978152+0000","last_fullsized":"2024-09-21T09:48:37.978152+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.886738+0000","last_change":"2024-09-21T09:47:24.056658+0000","last_active":"2024-09-21T09:48:37.886738+0000","last_peered":"2024-09-21T09:48:37.886738+0000","last_clean":"2024-09-21T09:48:37.886738+0000","last_became_active":"2024-09-21T09:47:24.056366+0000","last_became_peered":"2024-09-21T09:47:24.056366+0000","last_unstale":"2024-09-21T09:48:37.886738+0000","last_undegraded":"2024-09-21T09:48:37.886738+0000","last_fullsized":"2024-09-21T09:48:37.886738+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.427162+0000","last_change":"2024-09-21T09:48:37.861462+0000","last_active":"2024-09-21T09:48:41.427162+0000","last_peered":"2024-09-21T09:48:41.427162+0000","last_clean":"2024-09-21T09:48:41.427162+0000","last_became_active":"2024-09-21T09:48:33.677383+0000","last_became_peered":"2024-09-21T09:48:33.677383+0000","last_unstale":"2024-09-21T09:48:41.427162+0000","last_undegraded":"2024-09-21T09:48:41.427162+0000","last_fullsized":"2024-09-21T09:48:41.427162+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:37:12.106483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000471278,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861820+0000","last_change":"2024-09-21T09:47:20.861899+0000","last_active":"2024-09-21T09:48:37.861820+0000","last_peered":"2024-09-21T09:48:37.861820+0000","last_clean":"2024-09-21T09:48:37.861820+0000","last_became_active":"2024-09-21T09:47:20.861684+0000","last_became_peered":"2024-09-21T09:47:20.861684+0000","last_unstale":"2024-09-21T09:48:37.861820+0000","last_undegraded":"2024-09-21T09:48:37.861820+0000","last_fullsized":"2024-09-21T09:48:37.861820+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860233+0000","last_change":"2024-09-21T09:47:24.058235+0000","last_active":"2024-09-21T09:48:37.860233+0000","last_peered":"2024-09-21T09:48:37.860233+0000","last_clean":"2024-09-21T09:48:37.860233+0000","last_became_active":"2024-09-21T09:47:24.057972+0000","last_became_peered":"2024-09-21T09:47:24.057972+0000","last_unstale":"2024-09-21T09:48:37.860233+0000","last_undegraded":"2024-09-21T09:48:37.860233+0000","last_fullsized":"2024-09-21T09:48:37.860233+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.427876+0000","last_change":"2024-09-21T09:48:37.861803+0000","last_active":"2024-09-21T09:48:41.427876+0000","last_peered":"2024-09-21T09:48:41.427876+0000","last_clean":"2024-09-21T09:48:41.427876+0000","last_became_active":"2024-09-21T09:48:33.295964+0000","last_became_peered":"2024-09-21T09:48:33.295964+0000","last_unstale":"2024-09-21T09:48:41.427876+0000","last_undegraded":"2024-09-21T09:48:41.427876+0000","last_fullsized":"2024-09-21T09:48:41.427876+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:57:02.417883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010536689999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865760+0000","last_change":"2024-09-21T09:47:20.861547+0000","last_active":"2024-09-21T09:48:37.865760+0000","last_peered":"2024-09-21T09:48:37.865760+0000","last_clean":"2024-09-21T09:48:37.865760+0000","last_became_active":"2024-09-21T09:47:20.861254+0000","last_became_peered":"2024-09-21T09:47:20.861254+0000","last_unstale":"2024-09-21T09:48:37.865760+0000","last_undegraded":"2024-09-21T09:48:37.865760+0000","last_fullsized":"2024-09-21T09:48:37.865760+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860142+0000","last_change":"2024-09-21T09:47:24.060191+0000","last_active":"2024-09-21T09:48:37.860142+0000","last_peered":"2024-09-21T09:48:37.860142+0000","last_clean":"2024-09-21T09:48:37.860142+0000","last_became_active":"2024-09-21T09:47:24.059929+0000","last_became_peered":"2024-09-21T09:47:24.059929+0000","last_unstale":"2024-09-21T09:48:37.860142+0000","last_undegraded":"2024-09-21T09:48:37.860142+0000","last_fullsized":"2024-09-21T09:48:37.860142+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.2","version":"66'2","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.427101+0000","last_change":"2024-09-21T09:48:37.861137+0000","last_active":"2024-09-21T09:48:41.427101+0000","last_peered":"2024-09-21T09:48:41.427101+0000","last_clean":"2024-09-21T09:48:41.427101+0000","last_became_active":"2024-09-21T09:48:33.677309+0000","last_became_peered":"2024-09-21T09:48:33.677309+0000","last_unstale":"2024-09-21T09:48:41.427101+0000","last_undegraded":"2024-09-21T09:48:41.427101+0000","last_fullsized":"2024-09-21T09:48:41.427101+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:21:29.622677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056908600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861250+0000","last_change":"2024-09-21T09:47:20.862181+0000","last_active":"2024-09-21T09:48:37.861250+0000","last_peered":"2024-09-21T09:48:37.861250+0000","last_clean":"2024-09-21T09:48:37.861250+0000","last_became_active":"2024-09-21T09:47:20.861903+0000","last_became_peered":"2024-09-21T09:47:20.861903+0000","last_unstale":"2024-09-21T09:48:37.861250+0000","last_undegraded":"2024-09-21T09:48:37.861250+0000","last_fullsized":"2024-09-21T09:48:37.861250+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865932+0000","last_change":"2024-09-21T09:47:24.058849+0000","last_active":"2024-09-21T09:48:37.865932+0000","last_peered":"2024-09-21T09:48:37.865932+0000","last_clean":"2024-09-21T09:48:37.865932+0000","last_became_active":"2024-09-21T09:47:24.058564+0000","last_became_peered":"2024-09-21T09:47:24.058564+0000","last_unstale":"2024-09-21T09:48:37.865932+0000","last_undegraded":"2024-09-21T09:48:37.865932+0000","last_fullsized":"2024-09-21T09:48:37.865932+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"64'1","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.508707+0000","last_change":"2024-09-21T09:48:37.887211+0000","last_active":"2024-09-21T09:48:41.508707+0000","last_peered":"2024-09-21T09:48:41.508707+0000","last_clean":"2024-09-21T09:48:41.508707+0000","last_became_active":"2024-09-21T09:48:33.677589+0000","last_became_peered":"2024-09-21T09:48:33.677589+0000","last_unstale":"2024-09-21T09:48:41.508707+0000","last_undegraded":"2024-09-21T09:48:41.508707+0000","last_fullsized":"2024-09-21T09:48:41.508707+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:59:20.123742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040314399999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978674+0000","last_change":"2024-09-21T09:47:20.866885+0000","last_active":"2024-09-21T09:48:37.978674+0000","last_peered":"2024-09-21T09:48:37.978674+0000","last_clean":"2024-09-21T09:48:37.978674+0000","last_became_active":"2024-09-21T09:47:20.866436+0000","last_became_peered":"2024-09-21T09:47:20.866436+0000","last_unstale":"2024-09-21T09:48:37.978674+0000","last_undegraded":"2024-09-21T09:48:37.978674+0000","last_fullsized":"2024-09-21T09:48:37.978674+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861149+0000","last_change":"2024-09-21T09:47:24.060367+0000","last_active":"2024-09-21T09:48:37.861149+0000","last_peered":"2024-09-21T09:48:37.861149+0000","last_clean":"2024-09-21T09:48:37.861149+0000","last_became_active":"2024-09-21T09:47:24.060083+0000","last_became_peered":"2024-09-21T09:47:24.060083+0000","last_unstale":"2024-09-21T09:48:37.861149+0000","last_undegraded":"2024-09-21T09:48:37.861149+0000","last_fullsized":"2024-09-21T09:48:37.861149+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.655198+0000","last_change":"2024-09-21T09:48:38.080316+0000","last_active":"2024-09-21T09:48:41.655198+0000","last_peered":"2024-09-21T09:48:41.655198+0000","last_clean":"2024-09-21T09:48:41.655198+0000","last_became_active":"2024-09-21T09:48:33.298926+0000","last_became_peered":"2024-09-21T09:48:33.298926+0000","last_unstale":"2024-09-21T09:48:41.655198+0000","last_undegraded":"2024-09-21T09:48:41.655198+0000","last_fullsized":"2024-09-21T09:48:41.655198+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:41:34.343076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055753599999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861460+0000","last_change":"2024-09-21T09:47:20.858947+0000","last_active":"2024-09-21T09:48:37.861460+0000","last_peered":"2024-09-21T09:48:37.861460+0000","last_clean":"2024-09-21T09:48:37.861460+0000","last_became_active":"2024-09-21T09:47:20.858680+0000","last_became_peered":"2024-09-21T09:47:20.858680+0000","last_unstale":"2024-09-21T09:48:37.861460+0000","last_undegraded":"2024-09-21T09:48:37.861460+0000","last_fullsized":"2024-09-21T09:48:37.861460+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080299+0000","last_change":"2024-09-21T09:47:24.060498+0000","last_active":"2024-09-21T09:48:38.080299+0000","last_peered":"2024-09-21T09:48:38.080299+0000","last_clean":"2024-09-21T09:48:38.080299+0000","last_became_active":"2024-09-21T09:47:24.060179+0000","last_became_peered":"2024-09-21T09:47:24.060179+0000","last_unstale":"2024-09-21T09:48:38.080299+0000","last_undegraded":"2024-09-21T09:48:38.080299+0000","last_fullsized":"2024-09-21T09:48:38.080299+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-21T09:48:41.655530+0000","last_change":"2024-09-21T09:48:37.860425+0000","last_active":"2024-09-21T09:48:41.655530+0000","last_peered":"2024-09-21T09:48:41.655530+0000","last_clean":"2024-09-21T09:48:41.655530+0000","last_became_active":"2024-09-21T09:48:33.677625+0000","last_became_peered":"2024-09-21T09:48:33.677625+0000","last_unstale":"2024-09-21T09:48:41.655530+0000","last_undegraded":"2024-09-21T09:48:41.655530+0000","last_fullsized":"2024-09-21T09:48:41.655530+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:07:51.936200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00069840600000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.081020+0000","last_change":"2024-09-21T09:47:20.865797+0000","last_active":"2024-09-21T09:48:38.081020+0000","last_peered":"2024-09-21T09:48:38.081020+0000","last_clean":"2024-09-21T09:48:38.081020+0000","last_became_active":"2024-09-21T09:47:20.865466+0000","last_became_peered":"2024-09-21T09:47:20.865466+0000","last_unstale":"2024-09-21T09:48:38.081020+0000","last_undegraded":"2024-09-21T09:48:38.081020+0000","last_fullsized":"2024-09-21T09:48:38.081020+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887191+0000","last_change":"2024-09-21T09:47:24.058690+0000","last_active":"2024-09-21T09:48:37.887191+0000","last_peered":"2024-09-21T09:48:37.887191+0000","last_clean":"2024-09-21T09:48:37.887191+0000","last_became_active":"2024-09-21T09:47:24.058371+0000","last_became_peered":"2024-09-21T09:47:24.058371+0000","last_unstale":"2024-09-21T09:48:37.887191+0000","last_undegraded":"2024-09-21T09:48:37.887191+0000","last_fullsized":"2024-09-21T09:48:37.887191+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.8","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978109+0000","last_change":"2024-09-21T09:47:20.868994+0000","last_active":"2024-09-21T09:48:37.978109+0000","last_peered":"2024-09-21T09:48:37.978109+0000","last_clean":"2024-09-21T09:48:37.978109+0000","last_became_active":"2024-09-21T09:47:20.868686+0000","last_became_peered":"2024-09-21T09:47:20.868686+0000","last_unstale":"2024-09-21T09:48:37.978109+0000","last_undegraded":"2024-09-21T09:48:37.978109+0000","last_fullsized":"2024-09-21T09:48:37.978109+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860137+0000","last_change":"2024-09-21T09:47:24.055068+0000","last_active":"2024-09-21T09:48:37.860137+0000","last_peered":"2024-09-21T09:48:37.860137+0000","last_clean":"2024-09-21T09:48:37.860137+0000","last_became_active":"2024-09-21T09:47:24.054785+0000","last_became_peered":"2024-09-21T09:47:24.054785+0000","last_unstale":"2024-09-21T09:48:37.860137+0000","last_undegraded":"2024-09-21T09:48:37.860137+0000","last_fullsized":"2024-09-21T09:48:37.860137+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.9","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860615+0000","last_change":"2024-09-21T09:47:20.862674+0000","last_active":"2024-09-21T09:48:37.860615+0000","last_peered":"2024-09-21T09:48:37.860615+0000","last_clean":"2024-09-21T09:48:37.860615+0000","last_became_active":"2024-09-21T09:47:20.861988+0000","last_became_peered":"2024-09-21T09:47:20.861988+0000","last_unstale":"2024-09-21T09:48:37.860615+0000","last_undegraded":"2024-09-21T09:48:37.860615+0000","last_fullsized":"2024-09-21T09:48:37.860615+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.079906+0000","last_change":"2024-09-21T09:47:24.058826+0000","last_active":"2024-09-21T09:48:38.079906+0000","last_peered":"2024-09-21T09:48:38.079906+0000","last_clean":"2024-09-21T09:48:38.079906+0000","last_became_active":"2024-09-21T09:47:24.058565+0000","last_became_peered":"2024-09-21T09:47:24.058565+0000","last_unstale":"2024-09-21T09:48:38.079906+0000","last_undegraded":"2024-09-21T09:48:38.079906+0000","last_fullsized":"2024-09-21T09:48:38.079906+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.a","version":"0'0","reported_seq":24,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.860854+0000","last_change":"2024-09-21T09:47:20.861142+0000","last_active":"2024-09-21T09:48:37.860854+0000","last_peered":"2024-09-21T09:48:37.860854+0000","last_clean":"2024-09-21T09:48:37.860854+0000","last_became_active":"2024-09-21T09:47:20.860916+0000","last_became_peered":"2024-09-21T09:47:20.860916+0000","last_unstale":"2024-09-21T09:48:37.860854+0000","last_undegraded":"2024-09-21T09:48:37.860854+0000","last_fullsized":"2024-09-21T09:48:37.860854+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":31,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080038+0000","last_change":"2024-09-21T09:47:46.481406+0000","last_active":"2024-09-21T09:48:38.080038+0000","last_peered":"2024-09-21T09:48:38.080038+0000","last_clean":"2024-09-21T09:48:38.080038+0000","last_became_active":"2024-09-21T09:47:46.481006+0000","last_became_peered":"2024-09-21T09:47:46.481006+0000","last_unstale":"2024-09-21T09:48:38.080038+0000","last_undegraded":"2024-09-21T09:48:38.080038+0000","last_fullsized":"2024-09-21T09:48:38.080038+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:18:56.780478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.b","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978483+0000","last_change":"2024-09-21T09:47:20.865234+0000","last_active":"2024-09-21T09:48:37.978483+0000","last_peered":"2024-09-21T09:48:37.978483+0000","last_clean":"2024-09-21T09:48:37.978483+0000","last_became_active":"2024-09-21T09:47:20.863790+0000","last_became_peered":"2024-09-21T09:47:20.863790+0000","last_unstale":"2024-09-21T09:48:37.978483+0000","last_undegraded":"2024-09-21T09:48:37.978483+0000","last_fullsized":"2024-09-21T09:48:37.978483+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080437+0000","last_change":"2024-09-21T09:47:24.056216+0000","last_active":"2024-09-21T09:48:38.080437+0000","last_peered":"2024-09-21T09:48:38.080437+0000","last_clean":"2024-09-21T09:48:38.080437+0000","last_became_active":"2024-09-21T09:47:24.055928+0000","last_became_peered":"2024-09-21T09:47:24.055928+0000","last_unstale":"2024-09-21T09:48:38.080437+0000","last_undegraded":"2024-09-21T09:48:38.080437+0000","last_fullsized":"2024-09-21T09:48:38.080437+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.865838+0000","last_change":"2024-09-21T09:47:24.063533+0000","last_active":"2024-09-21T09:48:37.865838+0000","last_peered":"2024-09-21T09:48:37.865838+0000","last_clean":"2024-09-21T09:48:37.865838+0000","last_became_active":"2024-09-21T09:47:24.063293+0000","last_became_peered":"2024-09-21T09:47:24.063293+0000","last_unstale":"2024-09-21T09:48:37.865838+0000","last_undegraded":"2024-09-21T09:48:37.865838+0000","last_fullsized":"2024-09-21T09:48:37.865838+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978478+0000","last_change":"2024-09-21T09:47:20.864537+0000","last_active":"2024-09-21T09:48:37.978478+0000","last_peered":"2024-09-21T09:48:37.978478+0000","last_clean":"2024-09-21T09:48:37.978478+0000","last_became_active":"2024-09-21T09:47:20.864212+0000","last_became_peered":"2024-09-21T09:47:20.864212+0000","last_unstale":"2024-09-21T09:48:37.978478+0000","last_undegraded":"2024-09-21T09:48:37.978478+0000","last_fullsized":"2024-09-21T09:48:37.978478+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":18,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.861706+0000","last_change":"2024-09-21T09:47:46.774197+0000","last_active":"2024-09-21T09:48:37.861706+0000","last_peered":"2024-09-21T09:48:37.861706+0000","last_clean":"2024-09-21T09:48:37.861706+0000","last_became_active":"2024-09-21T09:47:46.773846+0000","last_became_peered":"2024-09-21T09:47:46.773846+0000","last_unstale":"2024-09-21T09:48:37.861706+0000","last_undegraded":"2024-09-21T09:48:37.861706+0000","last_fullsized":"2024-09-21T09:48:37.861706+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":26,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.978531+0000","last_change":"2024-09-21T09:47:20.866709+0000","last_active":"2024-09-21T09:48:37.978531+0000","last_peered":"2024-09-21T09:48:37.978531+0000","last_clean":"2024-09-21T09:48:37.978531+0000","last_became_active":"2024-09-21T09:47:20.866026+0000","last_became_peered":"2024-09-21T09:47:20.866026+0000","last_unstale":"2024-09-21T09:48:37.978531+0000","last_undegraded":"2024-09-21T09:48:37.978531+0000","last_fullsized":"2024-09-21T09:48:37.978531+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887503+0000","last_change":"2024-09-21T09:47:24.060801+0000","last_active":"2024-09-21T09:48:37.887503+0000","last_peered":"2024-09-21T09:48:37.887503+0000","last_clean":"2024-09-21T09:48:37.887503+0000","last_became_active":"2024-09-21T09:47:24.060536+0000","last_became_peered":"2024-09-21T09:47:24.060536+0000","last_unstale":"2024-09-21T09:48:37.887503+0000","last_undegraded":"2024-09-21T09:48:37.887503+0000","last_fullsized":"2024-09-21T09:48:37.887503+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"57'1","reported_seq":25,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080278+0000","last_change":"2024-09-21T09:47:20.864529+0000","last_active":"2024-09-21T09:48:38.080278+0000","last_peered":"2024-09-21T09:48:38.080278+0000","last_clean":"2024-09-21T09:48:38.080278+0000","last_became_active":"2024-09-21T09:47:20.864185+0000","last_became_peered":"2024-09-21T09:47:20.864185+0000","last_unstale":"2024-09-21T09:48:38.080278+0000","last_undegraded":"2024-09-21T09:48:38.080278+0000","last_fullsized":"2024-09-21T09:48:38.080278+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":21,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:38.080170+0000","last_change":"2024-09-21T09:47:24.065677+0000","last_active":"2024-09-21T09:48:38.080170+0000","last_peered":"2024-09-21T09:48:38.080170+0000","last_clean":"2024-09-21T09:48:38.080170+0000","last_became_active":"2024-09-21T09:47:24.065373+0000","last_became_peered":"2024-09-21T09:47:24.065373+0000","last_unstale":"2024-09-21T09:48:38.080170+0000","last_undegraded":"2024-09-21T09:48:38.080170+0000","last_fullsized":"2024-09-21T09:48:38.080170+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"57'4","reported_seq":28,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-21T09:48:37.887795+0000","last_change":"2024-09-21T09:47:20.865947+0000","last_active":"2024-09-21T09:48:37.887795+0000","last_peered":"2024-09-21T09:48:37.887795+0000","last_clean":"2024-09-21T09:48:37.887795+0000","last_became_active":"2024-09-21T09:47:20.865683+0000","last_became_peered":"2024-09-21T09:47:20.865683+0000","last_unstale":"2024-09-21T09:48:37.887795+0000","last_undegraded":"2024-09-21T09:48:37.887795+0000","last_fullsized":"2024-09-21T09:48:37.887795+0000","mapping_epoch":51,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":52,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":3128,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":48,"num_write_kb":16,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":9384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42,"ondisk_log_size":42,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430243,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1260,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1290240,"data_stored":900620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.48799999999999999,"15min":0.48899999999999999},"min":{"1min":0.32600000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.50600000000000001,"15min":0.504},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.71199999999999997,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.46000000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.433},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.53700000000000003,"15min":0.54000000000000004},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.52400000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52600000000000002,"15min":0.52200000000000002},"min":{"1min":0.40300000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.42099999999999999,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.53200000000000003,"15min":0.53600000000000003},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.66200000000000003,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53800000000000003,"15min":0.54400000000000004},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.56200000000000006}]},{"osd":4,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.502,"15min":0.51300000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.68000000000000005,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.433,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.57999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.626}]},{"osd":5,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.505,"15min":0.51700000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73199999999999998,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.55600000000000005,"15min":0.56399999999999995},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.57399999999999995,"15min":0.58599999999999997},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.71499999999999997,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.56699999999999995,"15min":0.58099999999999996},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.67500000000000004,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.65400000000000003}]}]},{"osd":6,"up_from":42,"seq":180388626471,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298612,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446540,"statfs":{"total":95995035648,"available":95689256960,"internally_reserved":0,"allocated":1286144,"data_stored":901811,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.47499999999999998,"15min":0.47099999999999997},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.67700000000000005,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.433},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.47199999999999998,"15min":0.46300000000000002},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.69199999999999995,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.52000000000000002,"15min":0.51600000000000001},"min":{"1min":0.40000000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.65600000000000003,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.52800000000000002,"15min":0.52300000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.71199999999999997,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.49199999999999999,"15min":0.48999999999999999},"min":{"1min":0.376,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.69999999999999996,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.51900000000000002,"15min":0.52100000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.61099999999999999,"5min":0.752,"15min":0.752},"last":0.47099999999999997}]},{"osd":3,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.50800000000000001,"15min":0.504},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.66600000000000004,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.50900000000000001,"15min":0.505},"min":{"1min":0.40400000000000003,"5min":0.373,"15min":0.373},"max":{"1min":0.68799999999999994,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.41599999999999998,"15min":0.41299999999999998},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.66300000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.432,"15min":0.42899999999999999},"min":{"1min":0.30399999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.59699999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:48:33 2024","interfaces":[{"interface":"back","average":{"1min":0.45500000000000002,"5min":0.42699999999999999,"15min":0.42199999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.63300000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.45700000000000002,"15min":0.45300000000000001},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.65200000000000002,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.52100000000000002,"15min":0.52600000000000002},"min":{"1min":0.36099999999999999,"5min":0.307,"15min":0.307},"max":{"1min":0.622,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.50900000000000001,"15min":0.51400000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.70399999999999996,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.61799999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822700,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298044,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447108,"statfs":{"total":95995035648,"available":95689838592,"internally_reserved":0,"allocated":704512,"data_stored":311796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48899999999999999,"15min":0.48299999999999998},"min":{"1min":0.33900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.59299999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.51400000000000001,"15min":0.51100000000000001},"min":{"1min":0.36399999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.65200000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.34200000000000003,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.69199999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.626},{"interface":"front","average":{"1min":0.503,"5min":0.502,"15min":0.498},"min":{"1min":0.32000000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.72099999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.59099999999999997}]},{"osd":2,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.52300000000000002,"15min":0.51100000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.38200000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.65700000000000003,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.69799999999999995}]},{"osd":3,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.52600000000000002,"15min":0.52700000000000002},"min":{"1min":0.40899999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.52400000000000002,"15min":0.51400000000000001},"min":{"1min":0.36799999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.73699999999999999,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Sep 21 09:49:17 2024","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.38400000000000001,"15min":0.36399999999999999},"min":{"1min":0.28399999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.38700000000000001,"15min":0.36599999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.59899999999999998,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 21 09:48:38 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.30099999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.59399999999999997,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.5,"15min":0.49199999999999999},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.53200000000000003,"15min":0.54000000000000004},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.70099999999999996,"5min":0.755,"15min":0.755},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.52600000000000002,"15min":0.53300000000000003},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.71299999999999997,"5min":0.752,"15min":0.752},"last":0.48699999999999999}]}]},{"osd":4,"up_from":30,"seq":128849018929,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298048,"kb_used_data":692,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447104,"statfs":{"total":95995035648,"available":95689834496,"internally_reserved":0,"allocated":708608,"data_stored":310375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:48:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.55700000000000005,"15min":0.57099999999999995},"min":{"1min":0.42899999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.64500000000000002,"5min":0.748,"15min":0.748},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.55200000000000005,"15min":0.55000000000000004},"min":{"1min":0.40400000000000003,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.63400000000000001}]},{"osd":1,"last update":"Sat Sep 21 09:48:41 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.53500000000000003,"15min":0.53300000000000003},"min":{"1min":0.41999999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.53600000000000003,"15min":0.53100000000000003},"min":{"1min":0.36599999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.61399999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:48:41 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54500000000000004,"15min":0.55000000000000004},"min":{"1min":0.39200000000000002,"5min":0.374,"15min":0.374},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.53100000000000003,"15min":0.52000000000000002},"min":{"1min":0.41399999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 21 09:48:41 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52400000000000002,"15min":0.52200000000000002},"min":{"1min":0.41999999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.55900000000000005,"15min":0.55400000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Sep 21 09:49:09 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.378,"15min":0.36199999999999999},"min":{"1min":0.25700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.60399999999999998,"5min":0.63800000000000001,"15min":0.63800000000000001},"last":0.436},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.40799999999999997,"15min":0.39100000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.52900000000000003,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.51100000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.42599999999999999,"15min":0.42399999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.47099999999999997,"15min":0.46999999999999997},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.66200000000000003,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 21 09:48:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":0.47499999999999998,"15min":0.48599999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.55500000000000005,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.53700000000000003,"15min":0.55100000000000005},"min":{"1min":0.32400000000000001,"5min":0.246,"15min":0.246},"max":{"1min":0.65300000000000002,"5min":0.749,"15min":0.749},"last":0.56899999999999995}]}]},{"osd":3,"up_from":25,"seq":107374182454,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298044,"kb_used_data":688,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447108,"statfs":{"total":95995035648,"available":95689838592,"internally_reserved":0,"allocated":704512,"data_stored":311791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.36899999999999999,"15min":0.39100000000000001},"min":{"1min":0.187,"5min":0.187,"15min":0.187},"max":{"1min":0.44500000000000001,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.34799999999999998,"5min":0.38800000000000001,"15min":0.40799999999999997},"min":{"1min":0.22500000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.55300000000000005,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.40699999999999997,"15min":0.437},"min":{"1min":0.23400000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.44,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.315},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.39900000000000002,"15min":0.42099999999999999},"min":{"1min":0.255,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.50700000000000001,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:49:31 2024","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.42799999999999999,"15min":0.44800000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.50900000000000001,"5min":0.66600000000000004,"15min":0.66600000000000004},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.371,"5min":0.41899999999999998,"15min":0.439},"min":{"1min":0.25,"5min":0.249,"15min":0.249},"max":{"1min":0.46400000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.37}]},{"osd":4,"last update":"Sat Sep 21 09:48:49 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58099999999999996},"min":{"1min":0.40500000000000003,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86199999999999999,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.57299999999999995,"15min":0.57499999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.71899999999999997,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.85499999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:49:08 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.623,"15min":0.65300000000000002},"min":{"1min":0.40400000000000003,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.71899999999999997,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.60499999999999998,"15min":0.621},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.66800000000000004,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.93200000000000005}]},{"osd":6,"last update":"Sat Sep 21 09:48:36 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.60099999999999998,"15min":0.60299999999999998},"min":{"1min":0.42599999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.70199999999999996,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.625,"5min":0.64500000000000002,"15min":0.64800000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.86499999999999999,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.81699999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:49:01 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.61299999999999999,"15min":0.622},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.71499999999999997,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.61699999999999999,"15min":0.627},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.72599999999999998,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.75800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411386,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298028,"kb_used_data":672,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447124,"statfs":{"total":95995035648,"available":95689854976,"internally_reserved":0,"allocated":688128,"data_stored":309660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.379,"15min":0.35999999999999999},"min":{"1min":0.26700000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.1259999999999999,"5min":2.1259999999999999,"15min":2.1259999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.41199999999999998,"15min":0.38900000000000001},"min":{"1min":0.28999999999999998,"5min":0.217,"15min":0.217},"max":{"1min":1.843,"5min":1.843,"15min":1.843},"last":0.27000000000000002}]},{"osd":1,"last update":"Sat Sep 21 09:49:06 2024","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.42299999999999999,"15min":0.41299999999999998},"min":{"1min":0.33100000000000002,"5min":0.254,"15min":0.254},"max":{"1min":1.9159999999999999,"5min":1.9159999999999999,"15min":1.9159999999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.45000000000000001,"15min":0.441},"min":{"1min":0.28699999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.052,"5min":2.052,"15min":2.052},"last":0.37}]},{"osd":3,"last update":"Sat Sep 21 09:49:25 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.46000000000000002,"15min":0.47199999999999998},"min":{"1min":0.28299999999999997,"5min":0.248,"15min":0.248},"max":{"1min":2.242,"5min":2.242,"15min":2.242},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.45400000000000001,"15min":0.46100000000000002},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.22},"max":{"1min":2.0030000000000001,"5min":2.0030000000000001,"15min":2.0030000000000001},"last":0.308}]},{"osd":4,"last update":"Sat Sep 21 09:48:46 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.58399999999999996,"15min":0.57899999999999996},"min":{"1min":0.45700000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.63},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.236,"5min":1.236,"15min":1.236},"last":0.67100000000000004}]},{"osd":5,"last update":"Sat Sep 21 09:49:12 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.58999999999999997,"15min":0.57799999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.60399999999999998,"15min":0.59599999999999997},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 21 09:49:34 2024","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.59199999999999997,"15min":0.57699999999999996},"min":{"1min":0.42899999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.70099999999999996,"5min":0.61899999999999999,"15min":0.60099999999999998},"min":{"1min":0.49199999999999999,"5min":0.435,"15min":0.435},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 21 09:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.66300000000000003,"5min":0.624,"15min":0.61799999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.198,"5min":1.198,"15min":1.198},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.622,"15min":0.61199999999999999},"min":{"1min":0.45700000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.61499999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574911,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298040,"kb_used_data":684,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447112,"statfs":{"total":95995035648,"available":95689842688,"internally_reserved":0,"allocated":700416,"data_stored":309839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 21 09:48:43 2024","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.36799999999999999,"15min":0.33000000000000002},"min":{"1min":0.223,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.36699999999999999,"15min":0.33100000000000002},"min":{"1min":0.26100000000000001,"5min":0.214,"15min":0.214},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Sep 21 09:49:04 2024","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.38100000000000001,"15min":0.377},"min":{"1min":0.252,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.71799999999999997,"5min":0.71799999999999997,"15min":0.71799999999999997},"last":0.312},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41399999999999998,"15min":0.40899999999999997},"min":{"1min":0.26300000000000001,"5min":0.251,"15min":0.251},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 21 09:49:25 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.441,"15min":0.44800000000000001},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.47399999999999998,"15min":0.47799999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 21 09:48:43 2024","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.55400000000000005,"15min":0.54400000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.58399999999999996,"15min":0.58599999999999997},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 21 09:49:13 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.42199999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.76900000000000002,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.58499999999999996,"15min":0.57999999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 21 09:48:35 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.60599999999999998,"15min":0.60399999999999998},"min":{"1min":0.44900000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.625,"15min":0.623},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.52300000000000002}]},{"osd":7,"last update":"Sat Sep 21 09:48:58 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.60699999999999998,"15min":0.60399999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.623},{"interface":"front","average":{"1min":0.63,"5min":0.622,"15min":0.62},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.94799999999999995,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.63400000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738436,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298620,"kb_used_data":1264,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446532,"statfs":{"total":95995035648,"available":95689248768,"internally_reserved":0,"allocated":1294336,"data_stored":900429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 21 09:48:46 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.35599999999999998,"15min":0.32200000000000001},"min":{"1min":0.214,"5min":0.188,"15min":0.188},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.38500000000000001,"15min":0.35299999999999998},"min":{"1min":0.26400000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.60999999999999999,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.29999999999999999}]},{"osd":2,"last update":"Sat Sep 21 09:49:03 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.39700000000000002,"15min":0.39100000000000001},"min":{"1min":0.23599999999999999,"5min":0.218,"15min":0.218},"max":{"1min":0.58399999999999996,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.39600000000000002,"15min":0.38600000000000001},"min":{"1min":0.26100000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.57999999999999996,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.378}]},{"osd":3,"last update":"Sat Sep 21 09:49:30 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.434,"15min":0.439},"min":{"1min":0.309,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.56399999999999995,"5min":0.72199999999999998,"15min":0.72199999999999998},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.42099999999999999,"15min":0.41299999999999998},"min":{"1min":0.25800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.315}]},{"osd":4,"last update":"Sat Sep 21 09:48:46 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56299999999999994,"15min":0.55300000000000005},"min":{"1min":0.35499999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.90000000000000002,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.57399999999999995,"15min":0.56799999999999995},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80100000000000005,"5min":0.871,"15min":0.871},"last":0.52400000000000002}]},{"osd":5,"last update":"Sat Sep 21 09:49:10 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.59599999999999997,"15min":0.61199999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68400000000000005,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.59299999999999997,"15min":0.58899999999999997},"min":{"1min":0.41399999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.89200000000000002,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 21 09:48:37 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.67900000000000005,"15min":0.68700000000000006},"min":{"1min":0.42199999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.83199999999999996,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.65700000000000003,"15min":0.66800000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.80000000000000004,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 21 09:48:57 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.64600000000000002,"15min":0.65200000000000002},"min":{"1min":0.39600000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.88100000000000001,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.629,"15min":0.64300000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.72699999999999998,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.61599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":2446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":2342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T09:49:42.385 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-21T09:49:42.385 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-21T09:49:42.385 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-21T09:49:42.386 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph health --format=json 2024-09-21T09:49:42.649 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:49:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:49:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:49:42.831 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:42 smithi033 ceph-mon[39045]: from='client.24647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:49:42.831 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:42 smithi033 ceph-mon[39045]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:42.831 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:42 smithi033 ceph-mon[27969]: from='client.24647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:49:42.832 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:42 smithi033 ceph-mon[27969]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:42.832 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:42 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:49:42.832 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:42 smithi033 systemd[1]: Stopped Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:42.832 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:42 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 1.026s CPU time. 2024-09-21T09:49:42.832 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:42 smithi033 systemd[1]: Starting Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:49:42.832 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:42 smithi033 podman[80200]: 2024-09-21 09:49:42.650240784 +0000 UTC m=+0.018769295 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-21T09:49:42.832 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:42 smithi033 podman[80200]: 2024-09-21 09:49:42.746311393 +0000 UTC m=+0.114839881 volume create 4eb5781d6440dee94e91d84e022a729658909cf6cc02c0dc0c83b3c1b8a3e1ff 2024-09-21T09:49:42.832 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:42 smithi033 podman[80200]: 2024-09-21 09:49:42.830049716 +0000 UTC m=+0.198578199 container create f5c871f256757f8d960aea793f66ff9256062d2ac60c69b2e150ec8eacc9e910 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:42 smithi105 ceph-mon[31286]: from='client.24647 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T09:49:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:42 smithi105 ceph-mon[31286]: pgmap v46: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:43.306 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:43.431 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 podman[80200]: 2024-09-21 09:49:43.12460629 +0000 UTC m=+0.493134791 container init f5c871f256757f8d960aea793f66ff9256062d2ac60c69b2e150ec8eacc9e910 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:43.431 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 podman[80200]: 2024-09-21 09:49:43.131691764 +0000 UTC m=+0.500220257 container start f5c871f256757f8d960aea793f66ff9256062d2ac60c69b2e150ec8eacc9e910 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:43.144Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:43.144Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:43.146Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.33 port=9094 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:43.147Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:43.170Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:43.171Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:43.173Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:43.173Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 bash[80200]: f5c871f256757f8d960aea793f66ff9256062d2ac60c69b2e150ec8eacc9e910 2024-09-21T09:49:43.432 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:43 smithi033 systemd[1]: Started Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:44.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:44.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:44.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[39045]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:49:44.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[39045]: Reconfiguring daemon prometheus.a on smithi105 2024-09-21T09:49:44.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[39045]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:44.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:44.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:44.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[27969]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:49:44.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[27969]: Reconfiguring daemon prometheus.a on smithi105 2024-09-21T09:49:44.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:44 smithi033 ceph-mon[27969]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:44.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:44 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:44.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:44 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:44.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:44 smithi105 ceph-mon[31286]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:49:44.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:44 smithi105 ceph-mon[31286]: Reconfiguring daemon prometheus.a on smithi105 2024-09-21T09:49:44.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:44 smithi105 ceph-mon[31286]: pgmap v47: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:45.170 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:49:45.170 INFO:teuthology.orchestra.run.smithi033.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-21T09:49:45.481 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:45 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:45.147Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000098449s 2024-09-21T09:49:45.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:45 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:45.482 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:45 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2399741296' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:49:45.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:45 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:45.482 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:45 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2399741296' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:49:45.490 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:45 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:45.490 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:45 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2399741296' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-21T09:49:45.884 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-21T09:49:45.884 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:49:46.143 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:46.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-mon[31286]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:46.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 systemd[1]: Stopping Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:49:46.560 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:46 smithi033 ceph-mon[27969]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:46.560 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:46 smithi033 ceph-mon[39045]: pgmap v48: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:46.974 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:49:46.974 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:49:46.974 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:49:46.974 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:49:46.974 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:49:46.974 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:49:46.974 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:49:46.974 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:49:46.975 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.689Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:49:46.975 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.690Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:49:46.975 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.690Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:49:46.975 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[61313]: ts=2024-09-21T09:49:46.690Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:49:46.975 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:46 smithi105 podman[64680]: 2024-09-21 09:49:46.700079984 +0000 UTC m=+0.120384864 container died cb180b46fbe0e1249659c10bd6a347d382455e31fc15360ef0cdf7b21b73fb10 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:47.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 podman[64680]: 2024-09-21 09:49:47.079452184 +0000 UTC m=+0.499757073 container cleanup cb180b46fbe0e1249659c10bd6a347d382455e31fc15360ef0cdf7b21b73fb10 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:47.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 bash[64680]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:49:47.706 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 podman[64697]: 2024-09-21 09:49:47.338887551 +0000 UTC m=+0.641440436 container remove cb180b46fbe0e1249659c10bd6a347d382455e31fc15360ef0cdf7b21b73fb10 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:47.706 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:49:47.706 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 systemd[1]: Stopped Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:47.706 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.088s CPU time. 2024-09-21T09:49:47.706 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 systemd[1]: Starting Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:49:47.980 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 podman[64794]: 2024-09-21 09:49:47.608549038 +0000 UTC m=+0.016642067 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-21T09:49:47.980 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 podman[64794]: 2024-09-21 09:49:47.714739623 +0000 UTC m=+0.122832652 container create 9b3306609c6022938b2b57a80fa81c4ed174a6c41ec949ac2ec0c3f8fbcf0af6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:47.981 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 podman[64794]: 2024-09-21 09:49:47.949745968 +0000 UTC m=+0.357839009 container init 9b3306609c6022938b2b57a80fa81c4ed174a6c41ec949ac2ec0c3f8fbcf0af6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:47.981 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 podman[64794]: 2024-09-21 09:49:47.954465816 +0000 UTC m=+0.362558849 container start 9b3306609c6022938b2b57a80fa81c4ed174a6c41ec949ac2ec0c3f8fbcf0af6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:49:48.008 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:49:48.008 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24323,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/125068843","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":125068843},{"type":"v1","addr":"172.21.15.33:6835","nonce":125068843}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:47:25.959476+0000","modified":"2024-09-21T09:47:39.447745+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24358},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24358":{"gid":24358,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.105:6833/2691139879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2691139879},{"type":"v1","addr":"172.21.15.105:6833","nonce":2691139879}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-21T09:49:48.008 INFO:teuthology.orchestra.run.smithi033.stderr:dumped fsmap epoch 7 2024-09-21T09:49:48.275 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.979Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.979Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.979Z caller=main.go:570 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.979Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.979Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.981Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.981Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.982Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.982Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.984Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.984Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.569µs 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.984Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.984Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-21T09:49:48.276 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.984Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-21T09:49:48.277 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.984Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.554µs wal_replay_duration=465.29µs wbl_replay_duration=87ns total_replay_duration=501.83µs 2024-09-21T09:49:48.277 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.985Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:49:48.277 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.985Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-21T09:49:48.277 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:47.985Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:49:48.277 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:48.000Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.866525ms db_storage=821ns remote_storage=1.392µs web_handler=330ns query_engine=536ns scrape=372.95µs scrape_sd=24.964µs notify=18.21µs notify_sd=5.912µs rules=14.209131ms tracing=5.922µs 2024-09-21T09:49:48.277 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:48.000Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-21T09:49:48.277 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:49:48.000Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:49:48.277 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:48 smithi105 bash[64794]: 9b3306609c6022938b2b57a80fa81c4ed174a6c41ec949ac2ec0c3f8fbcf0af6 2024-09-21T09:49:48.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:49:48 smithi105 systemd[1]: Started Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:49:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2776315786' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:49:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:49:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:49:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:49:48.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:49:48.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:49:48.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:49:48.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:48 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2776315786' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:49:48.680 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: pgmap v49: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2776315786' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:49:48.682 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:49:48.682 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.682 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:49:48.682 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:49:48.682 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:48.682 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:48] ENGINE Bus STOPPING 2024-09-21T09:49:48.706 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:49:48.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:49:48.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:48] ENGINE Bus STOPPED 2024-09-21T09:49:48.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:48] ENGINE Bus STARTING 2024-09-21T09:49:48.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:48] ENGINE Serving on http://:::9283 2024-09-21T09:49:48.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:48] ENGINE Bus STARTED 2024-09-21T09:49:48.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:48] ENGINE Bus STOPPING 2024-09-21T09:49:49.397 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[39045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:49:49.397 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[39045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:49:49.397 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[39045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:49:49.397 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[39045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:49:49.397 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[39045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[39045]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[27969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[27969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[27969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[27969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[27969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[27969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:49:49.398 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:49.399 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:49:49.399 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE Bus STOPPED 2024-09-21T09:49:49.399 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE Bus STARTING 2024-09-21T09:49:49.577 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:49.699 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE Serving on http://:::9283 2024-09-21T09:49:49.699 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE Bus STARTED 2024-09-21T09:49:49.699 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE Bus STOPPING 2024-09-21T09:49:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:49 smithi105 ceph-mon[31286]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:49:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:49 smithi105 ceph-mon[31286]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi033.front.sepia.ceph.com:9093"}]: dispatch 2024-09-21T09:49:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:49 smithi105 ceph-mon[31286]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:49:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:49 smithi105 ceph-mon[31286]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi105.front.sepia.ceph.com:9095"}]: dispatch 2024-09-21T09:49:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:49 smithi105 ceph-mon[31286]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:49:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:49 smithi105 ceph-mon[31286]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi105.front.sepia.ceph.com:3000"}]: dispatch 2024-09-21T09:49:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:49 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:49.954 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:49:49.954 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE Bus STOPPED 2024-09-21T09:49:49.954 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:49] ENGINE Bus STARTING 2024-09-21T09:49:50.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:50] ENGINE Serving on http://:::9283 2024-09-21T09:49:50.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:49:50] ENGINE Bus STARTED 2024-09-21T09:49:50.632 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:50 smithi033 ceph-mon[39045]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:50.632 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:50 smithi033 ceph-mon[27969]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:50.770 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:50 smithi105 ceph-mon[31286]: pgmap v50: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:51.323 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:49:51.323 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24323,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/125068843","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":125068843},{"type":"v1","addr":"172.21.15.33:6835","nonce":125068843}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:47:25.959476+0000","modified":"2024-09-21T09:47:39.447745+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24358},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24358":{"gid":24358,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.105:6833/2691139879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2691139879},{"type":"v1","addr":"172.21.15.105:6833","nonce":2691139879}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-21T09:49:51.323 INFO:teuthology.orchestra.run.smithi033.stderr:dumped fsmap epoch 7 2024-09-21T09:49:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:51 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3496285286' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:49:51.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:51 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3496285286' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:49:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:51 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3496285286' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:49:52.024 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T09:49:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:52 smithi033 ceph-mon[39045]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:52 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:52 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:52 smithi033 ceph-mon[27969]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:52 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:52 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:52 smithi105 ceph-mon[31286]: pgmap v51: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:52 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:52 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:52.795 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:53.233 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:49:53 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:49:53.148Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001652575s 2024-09-21T09:49:54.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[27969]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:54.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:54.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:54.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:54.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[39045]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:54.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:54.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:54.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:54 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:54 smithi105 ceph-mon[31286]: pgmap v52: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:54 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:54 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:54 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:49:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:54 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:49:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:54 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:54 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:49:55.346 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:49:55.346 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":67,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T09:48:41.423209+0000","last_up_change":"2024-09-21T09:46:55.803410+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:47:19.790061+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:47:22.976967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:48:32.209917+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6803","nonce":3277300496}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6805","nonce":3277300496}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6809","nonce":3277300496}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":3277300496},{"type":"v1","addr":"172.21.15.33:6807","nonce":3277300496}]},"public_addr":"172.21.15.33:6803/3277300496","cluster_addr":"172.21.15.33:6805/3277300496","heartbeat_back_addr":"172.21.15.33:6809/3277300496","heartbeat_front_addr":"172.21.15.33:6807/3277300496","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6811","nonce":2439650417}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6813","nonce":2439650417}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6817","nonce":2439650417}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2439650417},{"type":"v1","addr":"172.21.15.33:6815","nonce":2439650417}]},"public_addr":"172.21.15.33:6811/2439650417","cluster_addr":"172.21.15.33:6813/2439650417","heartbeat_back_addr":"172.21.15.33:6817/2439650417","heartbeat_front_addr":"172.21.15.33:6815/2439650417","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6819","nonce":3305661464}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6821","nonce":3305661464}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6825","nonce":3305661464}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":3305661464},{"type":"v1","addr":"172.21.15.33:6823","nonce":3305661464}]},"public_addr":"172.21.15.33:6819/3305661464","cluster_addr":"172.21.15.33:6821/3305661464","heartbeat_back_addr":"172.21.15.33:6825/3305661464","heartbeat_front_addr":"172.21.15.33:6823/3305661464","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6827","nonce":510343171}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6829","nonce":510343171}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6833","nonce":510343171}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":510343171},{"type":"v1","addr":"172.21.15.33:6831","nonce":510343171}]},"public_addr":"172.21.15.33:6827/510343171","cluster_addr":"172.21.15.33:6829/510343171","heartbeat_back_addr":"172.21.15.33:6833/510343171","heartbeat_front_addr":"172.21.15.33:6831/510343171","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6801","nonce":1234622360}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6803","nonce":1234622360}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6807","nonce":1234622360}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":1234622360},{"type":"v1","addr":"172.21.15.105:6805","nonce":1234622360}]},"public_addr":"172.21.15.105:6801/1234622360","cluster_addr":"172.21.15.105:6803/1234622360","heartbeat_back_addr":"172.21.15.105:6807/1234622360","heartbeat_front_addr":"172.21.15.105:6805/1234622360","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6809","nonce":1890006054}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6811","nonce":1890006054}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6815","nonce":1890006054}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1890006054},{"type":"v1","addr":"172.21.15.105:6813","nonce":1890006054}]},"public_addr":"172.21.15.105:6809/1890006054","cluster_addr":"172.21.15.105:6811/1890006054","heartbeat_back_addr":"172.21.15.105:6815/1890006054","heartbeat_front_addr":"172.21.15.105:6813/1890006054","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6817","nonce":592734975}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6819","nonce":592734975}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6823","nonce":592734975}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":592734975},{"type":"v1","addr":"172.21.15.105:6821","nonce":592734975}]},"public_addr":"172.21.15.105:6817/592734975","cluster_addr":"172.21.15.105:6819/592734975","heartbeat_back_addr":"172.21.15.105:6823/592734975","heartbeat_front_addr":"172.21.15.105:6821/592734975","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6825","nonce":455831701}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6827","nonce":455831701}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6831","nonce":455831701}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":455831701},{"type":"v1","addr":"172.21.15.105:6829","nonce":455831701}]},"public_addr":"172.21.15.105:6825/455831701","cluster_addr":"172.21.15.105:6827/455831701","heartbeat_back_addr":"172.21.15.105:6831/455831701","heartbeat_front_addr":"172.21.15.105:6829/455831701","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1a","mappings":[{"from":7,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.2f","mappings":[{"from":4,"to":3}]},{"pgid":"3.34","mappings":[{"from":4,"to":2}]},{"pgid":"3.39","mappings":[{"from":0,"to":1}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.33:0/150268354":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2645856759":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/3347076479":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/1156071037":"2024-09-22T09:48:29.268285+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6801/244283938":"2024-09-22T09:48:29.268285+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/244283938":"2024-09-22T09:48:29.268285+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":4,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T09:49:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:55 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2621310139' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:55.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:55 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2621310139' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:55.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:55 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2621310139' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-21T09:49:56.257 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T09:49:56.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:56 smithi033 ceph-mon[39045]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:56.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:56 smithi033 ceph-mon[27969]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:56 smithi105 ceph-mon[31286]: pgmap v53: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:56.956 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:49:58.867 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T09:49:58.867 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":7,"default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":24323,"name":"a","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.33:6835/125068843","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":125068843},{"type":"v1","addr":"172.21.15.33:6835","nonce":125068843}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"epoch":2}],"filesystems":[{"mdsmap":{"epoch":7,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:47:25.959476+0000","modified":"2024-09-21T09:47:39.447745+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"last_failure":0,"last_failure_osd_epoch":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"max_mds":1,"in":[0],"up":{"mds_0":24358},"failed":[],"damaged":[],"stopped":[],"info":{"gid_24358":{"gid":24358,"name":"b","rank":0,"incarnation":4,"state":"up:active","state_seq":5,"addr":"172.21.15.105:6833/2691139879","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2691139879},{"type":"v1","addr":"172.21.15.105:6833","nonce":2691139879}]},"join_fscid":-1,"export_targets":[],"features":4540138320759226367,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","standby_count_wanted":0},"id":1}]} 2024-09-21T09:49:58.867 INFO:teuthology.orchestra.run.smithi033.stderr:dumped fsmap epoch 7 2024-09-21T09:49:58.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:58 smithi033 ceph-mon[39045]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:58.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:58 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:58.982 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:58 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:58.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:58 smithi033 ceph-mon[27969]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:58.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:58 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:58.982 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:58 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:58.983 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:49:58 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:49:58] "GET /metrics HTTP/1.1" 200 236509 "" "Prometheus/2.43.0" 2024-09-21T09:49:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:58 smithi105 ceph-mon[31286]: pgmap v54: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:49:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:58 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:58 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:49:59.682 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: mds map: {'epoch': 7, 'flags': 18, 'flags_state': {'joinable': True, 'allow_snaps': True, 'allow_multimds_snaps': True, 'allow_standby_replay': False}, 'ever_allowed_features': 0, 'explicitly_allowed_features': 0, 'created': '2024-09-21T09:47:25.959476+0000', 'modified': '2024-09-21T09:47:39.447745+0000', 'tableserver': 0, 'root': 0, 'session_timeout': 60, 'session_autoclose': 300, 'required_client_features': {}, 'max_file_size': 1099511627776, 'last_failure': 0, 'last_failure_osd_epoch': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}, 'max_mds': 1, 'in': [0], 'up': {'mds_0': 24358}, 'failed': [], 'damaged': [], 'stopped': [], 'info': {'gid_24358': {'gid': 24358, 'name': 'b', 'rank': 0, 'incarnation': 4, 'state': 'up:active', 'state_seq': 5, 'addr': '172.21.15.105:6833/2691139879', 'addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.105:6832', 'nonce': 2691139879}, {'type': 'v1', 'addr': '172.21.15.105:6833', 'nonce': 2691139879}]}, 'join_fscid': -1, 'export_targets': [], 'features': 4540138320759226367, 'flags': 0, 'compat': {'compat': {}, 'ro_compat': {}, 'incompat': {'feature_1': 'base v0.20', 'feature_2': 'client writeable ranges', 'feature_3': 'default file layouts on dirs', 'feature_4': 'dir inode in separate object', 'feature_5': 'mds uses versioned encoding', 'feature_6': 'dirfrag is stored in omap', 'feature_7': 'mds uses inline data', 'feature_8': 'no anchor table', 'feature_9': 'file layout v2', 'feature_10': 'snaprealm v2'}}}}, 'data_pools': [3], 'metadata_pool': 2, 'enabled': True, 'fs_name': 'cephfs', 'balancer': '', 'standby_count_wanted': 0} 2024-09-21T09:49:59.682 DEBUG:tasks.cephfs.filesystem:are_daemons_healthy: 1/1 2024-09-21T09:49:59.682 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph tell mds.b status 2024-09-21T09:49:59.907 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:59 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2871945707' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:49:59.907 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:49:59 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:49:59.907 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:59 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2871945707' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:49:59.907 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:49:59 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:00.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:59 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2871945707' entity='client.admin' cmd=[{"prefix": "fs dump", "format": "json"}]: dispatch 2024-09-21T09:50:00.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:49:59 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:00.942 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:00 smithi033 ceph-mon[39045]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:00 smithi033 ceph-mon[39045]: overall HEALTH_OK 2024-09-21T09:50:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:00 smithi033 ceph-mon[27969]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:00 smithi033 ceph-mon[27969]: overall HEALTH_OK 2024-09-21T09:50:01.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:00 smithi105 ceph-mon[31286]: pgmap v55: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:00 smithi105 ceph-mon[31286]: overall HEALTH_OK 2024-09-21T09:50:02.202 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.202 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.202 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:02.202 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:02.202 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.202 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[27969]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:02.203 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.203 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.203 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:02.203 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:02.203 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.203 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:01 smithi033 ceph-mon[39045]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:01 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:01 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:01 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:01 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:01 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:01 smithi105 ceph-mon[31286]: pgmap v56: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:02.746 INFO:teuthology.orchestra.run.smithi033.stderr:2024-09-21T09:50:02.742+0000 7f08e4ff9640 0 client.14784 ms_handle_reset on v2:172.21.15.105:6832/2691139879 2024-09-21T09:50:02.764 INFO:teuthology.orchestra.run.smithi033.stderr:2024-09-21T09:50:02.761+0000 7f08e4ff9640 0 client.14790 ms_handle_reset on v2:172.21.15.105:6832/2691139879 2024-09-21T09:50:02.764 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:50:02.764 INFO:teuthology.orchestra.run.smithi033.stdout: "cluster_fsid": "a29e8460-77fd-11ef-baf6-efdc52797490", 2024-09-21T09:50:02.764 INFO:teuthology.orchestra.run.smithi033.stdout: "whoami": 0, 2024-09-21T09:50:02.764 INFO:teuthology.orchestra.run.smithi033.stdout: "id": 24358, 2024-09-21T09:50:02.764 INFO:teuthology.orchestra.run.smithi033.stdout: "want_state": "up:active", 2024-09-21T09:50:02.764 INFO:teuthology.orchestra.run.smithi033.stdout: "state": "up:active", 2024-09-21T09:50:02.765 INFO:teuthology.orchestra.run.smithi033.stdout: "fs_name": "cephfs", 2024-09-21T09:50:02.765 INFO:teuthology.orchestra.run.smithi033.stdout: "rank_uptime": 156.712652437, 2024-09-21T09:50:02.765 INFO:teuthology.orchestra.run.smithi033.stdout: "mdsmap_epoch": 7, 2024-09-21T09:50:02.765 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_epoch": 67, 2024-09-21T09:50:02.765 INFO:teuthology.orchestra.run.smithi033.stdout: "osdmap_epoch_barrier": 57, 2024-09-21T09:50:02.765 INFO:teuthology.orchestra.run.smithi033.stdout: "uptime": 171.09419248399999 2024-09-21T09:50:02.765 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:50:02.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:50:02 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:50:03.058 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:02 smithi033 ceph-mon[27969]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-21T09:50:03.058 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:02 smithi033 ceph-mon[39045]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-21T09:50:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:02 smithi105 ceph-mon[31286]: mgrmap e21: y(active, since 92s), standbys: x 2024-09-21T09:50:03.387 INFO:tasks.cephadm:Setup complete, yielding 2024-09-21T09:50:03.388 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:50:03.396 INFO:teuthology.task.print:**** done end installing quincy cephadm ... 2024-09-21T09:50:03.396 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:50:03.446 INFO:teuthology.task.print:**** done start cephadm.shell ceph config set mgr... 2024-09-21T09:50:03.447 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-21T09:50:03.493 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-21T09:50:03.493 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- bash -c 'ceph config set mgr mgr/cephadm/use_repo_digest true --force' 2024-09-21T09:50:03.763 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:50:04.203 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:03 smithi033 ceph-mon[39045]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:04.203 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:03 smithi033 ceph-mon[27969]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:03 smithi105 ceph-mon[31286]: pgmap v57: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:05.838 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:05 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:50:05] ENGINE Bus STOPPING 2024-09-21T09:50:06.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:05 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:50:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:50:06.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:05 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:50:05] ENGINE Bus STOPPED 2024-09-21T09:50:06.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:05 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:50:05] ENGINE Bus STARTING 2024-09-21T09:50:06.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:50:06] ENGINE Serving on http://:::9283 2024-09-21T09:50:06.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:50:06] ENGINE Bus STARTED 2024-09-21T09:50:06.148 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:50:06.157 INFO:teuthology.task.print:**** done cephadm.shell ceph config set mgr... 2024-09-21T09:50:06.157 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:50:06.217 INFO:teuthology.task.print:**** done start telemetry quincy... 2024-09-21T09:50:06.217 INFO:teuthology.run_tasks:Running task workunit... 2024-09-21T09:50:06.237 INFO:tasks.workunit:Pulling workunits from ref 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:50:06.237 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-21T09:50:06.237 DEBUG:teuthology.orchestra.run.smithi033:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:50:06.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T09:50:06.255 INFO:teuthology.orchestra.run.smithi033.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-21T09:50:06.255 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:50:06.311 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-21T09:50:06.312 DEBUG:teuthology.orchestra.run.smithi033:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-21T09:50:06.368 INFO:tasks.workunit:timeout=3h 2024-09-21T09:50:06.369 INFO:tasks.workunit:cleanup=True 2024-09-21T09:50:06.369 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 && git checkout 62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:50:06.426 INFO:tasks.workunit.client.0.smithi033.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-21T09:50:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:06 smithi105 ceph-mon[31286]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:06 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' 2024-09-21T09:50:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:06 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:06 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:06 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:06 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[39045]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[27969]: pgmap v58: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:50:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:50:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:50:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:06 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:50:08.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:08 smithi033 ceph-mon[39045]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:08 smithi033 ceph-mon[27969]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:08 smithi105 ceph-mon[31286]: pgmap v59: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:09.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:08] "GET /metrics HTTP/1.1" 200 236509 "" "Prometheus/2.43.0" 2024-09-21T09:50:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:10 smithi105 ceph-mon[31286]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:10.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:10 smithi033 ceph-mon[39045]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:10.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:10 smithi033 ceph-mon[27969]: pgmap v60: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:12 smithi105 ceph-mon[31286]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:12.781 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:50:12 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:50:12.798 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:12 smithi033 ceph-mon[39045]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:12.799 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:12 smithi033 ceph-mon[27969]: pgmap v61: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-21T09:50:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:14 smithi033 ceph-mon[27969]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:14 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:14 smithi033 ceph-mon[39045]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:14 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:14 smithi105 ceph-mon[31286]: pgmap v62: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:14 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:16 smithi033 ceph-mon[39045]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:16 smithi033 ceph-mon[27969]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:16 smithi105 ceph-mon[31286]: pgmap v63: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:18.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:18 smithi033 ceph-mon[39045]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:18 smithi033 ceph-mon[27969]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:18 smithi105 ceph-mon[31286]: pgmap v64: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:19.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:18] "GET /metrics HTTP/1.1" 200 236520 "" "Prometheus/2.43.0" 2024-09-21T09:50:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:20 smithi033 ceph-mon[39045]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:20 smithi033 ceph-mon[27969]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:20 smithi105 ceph-mon[31286]: pgmap v65: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:22 smithi105 ceph-mon[31286]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:22.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:50:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:50:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:22 smithi033 ceph-mon[39045]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:22 smithi033 ceph-mon[27969]: pgmap v66: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:24.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:24 smithi033 ceph-mon[27969]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:24 smithi033 ceph-mon[39045]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:25.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:24 smithi105 ceph-mon[31286]: pgmap v67: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:26 smithi033 ceph-mon[39045]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:26 smithi033 ceph-mon[27969]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:26 smithi105 ceph-mon[31286]: pgmap v68: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:28.920 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:28 smithi033 ceph-mon[27969]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:28.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:28 smithi033 ceph-mon[39045]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:28 smithi105 ceph-mon[31286]: pgmap v69: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:29.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:28] "GET /metrics HTTP/1.1" 200 236520 "" "Prometheus/2.43.0" 2024-09-21T09:50:29.802 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:30 smithi033 ceph-mon[39045]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:30 smithi033 ceph-mon[27969]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:31.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:30 smithi105 ceph-mon[31286]: pgmap v70: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:32 smithi105 ceph-mon[31286]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:32.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:50:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:50:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:32 smithi033 ceph-mon[39045]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:32 smithi033 ceph-mon[27969]: pgmap v71: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:34 smithi033 ceph-mon[39045]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:34.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:34 smithi033 ceph-mon[27969]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:34 smithi105 ceph-mon[31286]: pgmap v72: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:36 smithi033 ceph-mon[39045]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:36 smithi033 ceph-mon[27969]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:36 smithi105 ceph-mon[31286]: pgmap v73: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:38.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:38 smithi033 ceph-mon[39045]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:38.920 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:38 smithi033 ceph-mon[27969]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:38 smithi105 ceph-mon[31286]: pgmap v74: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:39.245 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:38 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:38] "GET /metrics HTTP/1.1" 200 236506 "" "Prometheus/2.43.0" 2024-09-21T09:50:40.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:40 smithi033 ceph-mon[39045]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:40.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:40 smithi033 ceph-mon[27969]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:40 smithi105 ceph-mon[31286]: pgmap v75: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:42 smithi105 ceph-mon[31286]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:42.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:50:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:50:42.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:42 smithi033 ceph-mon[39045]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:42.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:42 smithi033 ceph-mon[27969]: pgmap v76: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:44 smithi033 ceph-mon[39045]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:44 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:44 smithi033 ceph-mon[27969]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:44 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:45.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:44 smithi105 ceph-mon[31286]: pgmap v77: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:45.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:44 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:46 smithi033 ceph-mon[39045]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:46 smithi033 ceph-mon[27969]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:46 smithi105 ceph-mon[31286]: pgmap v78: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:48.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:48 smithi033 ceph-mon[39045]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:48.920 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:48 smithi033 ceph-mon[27969]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:48 smithi105 ceph-mon[31286]: pgmap v79: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:49.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:48] "GET /metrics HTTP/1.1" 200 236503 "" "Prometheus/2.43.0" 2024-09-21T09:50:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:50 smithi033 ceph-mon[27969]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:50 smithi033 ceph-mon[39045]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:50 smithi105 ceph-mon[31286]: pgmap v80: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:52.598 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:50:52 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:50:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:52 smithi033 ceph-mon[39045]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:52 smithi033 ceph-mon[27969]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:52 smithi105 ceph-mon[31286]: pgmap v81: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:54.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:54 smithi033 ceph-mon[39045]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:54.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:54 smithi033 ceph-mon[27969]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:54 smithi105 ceph-mon[31286]: pgmap v82: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:56 smithi033 ceph-mon[39045]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:56.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:56 smithi033 ceph-mon[27969]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:56 smithi105 ceph-mon[31286]: pgmap v83: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:58.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:58 smithi033 ceph-mon[39045]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:58.920 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:50:58 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:50:58] "GET /metrics HTTP/1.1" 200 236503 "" "Prometheus/2.43.0" 2024-09-21T09:50:58.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:58 smithi033 ceph-mon[27969]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:58 smithi105 ceph-mon[31286]: pgmap v84: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:50:59.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:50:59 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:50:59 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:50:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:50:59 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:00 smithi033 ceph-mon[39045]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:00 smithi033 ceph-mon[27969]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:01.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:00 smithi105 ceph-mon[31286]: pgmap v85: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:02.620 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:51:02 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:51:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:02 smithi033 ceph-mon[39045]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:02 smithi033 ceph-mon[27969]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:02 smithi105 ceph-mon[31286]: pgmap v86: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:04.702 INFO:tasks.workunit.client.0.smithi033.stderr:Updating files: 95% (12267/12896) Updating files: 96% (12381/12896) Updating files: 97% (12510/12896) Updating files: 98% (12639/12896) Updating files: 99% (12768/12896) Updating files: 100% (12896/12896) Updating files: 100% (12896/12896), done. 2024-09-21T09:51:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:04 smithi033 ceph-mon[39045]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:04 smithi033 ceph-mon[27969]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:05.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:04 smithi105 ceph-mon[31286]: pgmap v87: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:05.269 INFO:tasks.workunit.client.0.smithi033.stderr:Note: switching to '62da54b4b83a96ca9976b7e1d0c8272a564a2208'. 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr:changes and commit them, and you can discard any commits you make in this 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr:state without impacting any branches by switching back to a branch. 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr:If you want to create a new branch to retain commits you create, you may 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr:do so (now or later) by using -c with the switch command. Example: 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr: git switch -c 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr:Or undo this operation with: 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:05.270 INFO:tasks.workunit.client.0.smithi033.stderr: git switch - 2024-09-21T09:51:05.271 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:05.271 INFO:tasks.workunit.client.0.smithi033.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2024-09-21T09:51:05.271 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:05.271 INFO:tasks.workunit.client.0.smithi033.stderr:HEAD is now at 62da54b4b83 Merge pull request #59904 from mcv21/patch-2 2024-09-21T09:51:05.272 DEBUG:teuthology.orchestra.run.smithi033:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-21T09:51:05.335 INFO:tasks.workunit.client.0.smithi033.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-21T09:51:05.337 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T09:51:05.338 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-21T09:51:06.019 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-21T09:51:06.178 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-21T09:51:06.218 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T09:51:06.220 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T09:51:06.220 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-21T09:51:06.260 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T09:51:06.263 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T09:51:06.263 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-21T09:51:06.320 INFO:tasks.workunit:Running workunits matching test_telemetry_quincy.sh on client.0... 2024-09-21T09:51:06.321 INFO:tasks.workunit:Running workunit test_telemetry_quincy.sh... 2024-09-21T09:51:06.322 DEBUG:teuthology.orchestra.run.smithi033:workunit test test_telemetry_quincy.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=62da54b4b83a96ca9976b7e1d0c8272a564a2208 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/test_telemetry_quincy.sh 2024-09-21T09:51:06.384 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph config set mgr mgr/telemetry/channel_ident true 2024-09-21T09:51:06.770 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph config set mgr mgr/telemetry/organization ceph-qa 2024-09-21T09:51:06.931 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:06] ENGINE Bus STOPPING 2024-09-21T09:51:06.932 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:51:06.932 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:06] ENGINE Bus STOPPED 2024-09-21T09:51:06.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[39045]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:06.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:06.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:06.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:06.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:06.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[27969]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:06.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:06.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:06.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:06.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:06 smithi105 ceph-mon[31286]: pgmap v88: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:06 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:06 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:06 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:06 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:07.141 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph config set mgr mgr/telemetry/description 'upgrade test cluster' 2024-09-21T09:51:07.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:06] ENGINE Bus STARTING 2024-09-21T09:51:07.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE Serving on http://:::9283 2024-09-21T09:51:07.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE Bus STARTED 2024-09-21T09:51:07.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE Bus STOPPING 2024-09-21T09:51:07.511 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph telemetry preview 2024-09-21T09:51:07.974 INFO:tasks.workunit.client.0.smithi033.stdout:{ 2024-09-21T09:51:07.974 INFO:tasks.workunit.client.0.smithi033.stdout: "balancer": { 2024-09-21T09:51:07.974 INFO:tasks.workunit.client.0.smithi033.stdout: "active": true, 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: "mode": "" 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: "channels": [ 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: "basic", 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: "crash", 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: "device", 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: "ident" 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:07.975 INFO:tasks.workunit.client.0.smithi033.stdout: "channels_available": [ 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "basic", 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "ident", 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "crash", 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "device", 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "perf" 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "collections_available": [ 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_base", 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "device_base", 2024-09-21T09:51:07.976 INFO:tasks.workunit.client.0.smithi033.stdout: "crash_base", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "ident_base", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_perf", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_mds_metadata", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_usage", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_usage_by_class", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_rook_v01", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_memory_metrics", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_options_bluestore" 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "collections_opted_in": [ 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_base", 2024-09-21T09:51:07.977 INFO:tasks.workunit.client.0.smithi033.stdout: "device_base", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "crash_base", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "ident_base", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_perf", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_mds_metadata", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_usage", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_usage_by_class", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_rook_v01", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_memory_metrics", 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_options_bluestore" 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "config": { 2024-09-21T09:51:07.978 INFO:tasks.workunit.client.0.smithi033.stdout: "active_changed": [ 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_debug", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_service_ticket_ttl", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "bdev_debug_aio", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "client_check_pool_perm", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "client_mount_gid", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "client_mount_uid", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "client_quota", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "container_image", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "daemonize", 2024-09-21T09:51:07.979 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mgr", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mon", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_ms", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_osd", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_paxos", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "keyring", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_block_size", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_cache_size", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_compression", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_log", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_write_buffer_size", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_file", 2024-09-21T09:51:07.980 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_journald", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_stderr", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_allow_pool_delete", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_clock_drift_allowed", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_file", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_journald", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_stderr", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_data_avail_warn", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_election_default_strategy", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_host", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_max_pg_per_osd", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:51:07.981 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:51:07.982 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_bug", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_old_message", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "no_config_file", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_default_list", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_load_list", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_op_order", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_shutdown", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:51:07.983 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_profile", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_lim", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_res", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target", 2024-09-21T09:51:07.984 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target_autotune", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue_cut_off", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recover_clone_overlap", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recovery_max_chunk", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_load_threshold", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_max_interval", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "public_addr", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "public_addrv", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "public_network", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd_default_features", 2024-09-21T09:51:07.985 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd_qos_exclude_ops", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "setgroup", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "setuser" 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster_changed": [ 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_debug", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_service_ticket_ttl", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "bdev_debug_aio", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "container_image", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:51:07.986 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mgr", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mon", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_ms", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_osd", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_paxos", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_file", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_journald", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_stderr", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_join_fs", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/allow_ptrace", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/container_init", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/migration_current", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/use_repo_digest", 2024-09-21T09:51:07.987 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/ssl_server_port", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/orchestrator/orchestrator", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/channel_ident", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/description", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/organization", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_allow_pool_delete", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_clock_drift_allowed", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_file", 2024-09-21T09:51:07.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_data_avail_warn", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_election_default_strategy", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_max_pg_per_osd", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:51:07.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_bug", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_old_message", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_default_list", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_load_list", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_op_order", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_shutdown", 2024-09-21T09:51:07.990 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_profile", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target_autotune", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue_cut_off", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recover_clone_overlap", 2024-09-21T09:51:07.991 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recovery_max_chunk", 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_load_threshold", 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_max_interval", 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "public_network", 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_cache_enabled", 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_enable_ops_log", 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_enable_usage_log" 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "contact": null, 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "crashes": [], 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "created": "2024-09-21T09:41:48.442447Z", 2024-09-21T09:51:07.992 INFO:tasks.workunit.client.0.smithi033.stdout: "crush": { 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_algs": { 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "straw2": 3 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_sizes": { 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "2": 1, 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "4": 2 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_types": { 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "1": 2, 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "11": 1 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "compat_weight_set": false, 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: "device_classes": [ 2024-09-21T09:51:07.993 INFO:tasks.workunit.client.0.smithi033.stdout: 8 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_buckets": 6, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_devices": 8, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_rules": 1, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_types": 12, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_weight_sets": 0, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "tunables": { 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "allowed_bucket_algs": 54, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_local_fallback_tries": 0, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_local_tries": 0, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_total_tries": 50, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_descend_once": 1, 2024-09-21T09:51:07.994 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_stable": 1, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_vary_r": 1, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v2_rules": 0, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v3_rules": 0, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v4_buckets": 1, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v5_rules": 0, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "legacy_tunables": 0, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "minimum_required_version": "jewel", 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "optimal_tunables": 1, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "profile": "jewel", 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables": 1, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables2": 1, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables3": 1, 2024-09-21T09:51:07.995 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables5": 1, 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "straw_calc_version": 1 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "description": "upgrade test cluster", 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "fs": { 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "feature_flags": { 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "enable_multiple": true, 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "ever_enabled_multiple": true 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "filesystems": [ 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:07.996 INFO:tasks.workunit.client.0.smithi033.stdout: "approx_ctime": "2024-09", 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "balancer_enabled": false, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_caps": 0, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_dns": 10, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_inos": 13, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_subtrees": 2, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "ever_allowed_features": 0, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "explicitly_allowed_features": 0, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "files": 0, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "max_mds": 1, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "num_data_pools": 1, 2024-09-21T09:51:07.997 INFO:tasks.workunit.client.0.smithi033.stdout: "num_in": 1, 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: "num_mds": 1, 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": 0, 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: "num_standby_replay": 0, 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: "num_up": 1, 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: "snaps": 0, 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: "standby_count_wanted": 0 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:07.998 INFO:tasks.workunit.client.0.smithi033.stdout: "num_standby_mds": 1, 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: "total_num_mds": 2 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: "hosts": { 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num": 2, 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mds": 2, 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mgr": 2, 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mon": 2, 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_osd": 2 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:07.999 INFO:tasks.workunit.client.0.smithi033.stdout: "leaderboard": false, 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "license": "sharing-1-0", 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "metadata": { 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 2 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:08.000 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 1, 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 2 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 2 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.001 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 2 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 2 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 2 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.002 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 3 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2, 2024-09-21T09:51:08.003 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 3 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 3 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 3 2024-09-21T09:51:08.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 3 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 3 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:08.005 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 8 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 4, 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 4 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.006 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 8 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 8 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 8 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.007 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 8 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 8 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_objectstore": { 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": 8 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: "rotational": { 2024-09-21T09:51:08.008 INFO:tasks.workunit.client.0.smithi033.stdout: "0": 8 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: "features": { 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: "optional": [], 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: "persistent": [ 2024-09-21T09:51:08.009 INFO:tasks.workunit.client.0.smithi033.stdout: "kraken", 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: "luminous", 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: "mimic", 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap-prune", 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: "nautilus", 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: "octopus", 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: "pacific", 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: "elector-pinging", 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: "quincy" 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:08.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: "ipv4_addr_mons": 6, 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: "ipv6_addr_mons": 0, 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: "min_mon_release": 17, 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: "v1_addr_mons": 3, 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: "v2_addr_mons": 3 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: "organization": "ceph-qa", 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:08.011 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster_network": false, 2024-09-21T09:51:08.012 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 8, 2024-09-21T09:51:08.012 INFO:tasks.workunit.client.0.smithi033.stdout: "require_min_compat_client": "luminous", 2024-09-21T09:51:08.012 INFO:tasks.workunit.client.0.smithi033.stdout: "require_osd_release": "quincy" 2024-09-21T09:51:08.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "pools": [ 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr" 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 1, 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 1, 2024-09-21T09:51:08.013 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 1, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 1781760, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 1781760, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 2, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:08.014 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 126, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 236544, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 590368, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 590368, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 1771104, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 113, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 1667072 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:08.015 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 32, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 32, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 2, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 98304, 2024-09-21T09:51:08.016 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 98304, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 22, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 3410, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 3410, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:08.017 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 10230, 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 51, 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 19456 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:08.018 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 64, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 64, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 3, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 0, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 0, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 0, 2024-09-21T09:51:08.019 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 0.0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 0 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:08.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd" 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 8, 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 8, 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 4, 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:08.021 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 12288, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 12288, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 2, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 1.689964257423071e-08, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:08.022 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 19, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 19, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 57, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 2, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 2048 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd": { 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: "mirroring_by_pool": [ 2024-09-21T09:51:08.023 INFO:tasks.workunit.client.0.smithi033.stdout: false 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "num_images_by_pool": [ 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pools": 1 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "report_id": "baa922a1-6d4a-42b6-88d5-8a719a132b6b", 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "report_timestamp": "2024-09-21T09:51:07.967049", 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "report_version": 1, 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "rook": { 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster": { 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "external-mode": null, 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:08.024 INFO:tasks.workunit.client.0.smithi033.stdout: "allow-multiple-per-node": null, 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "count": null, 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "max-id": null, 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "pvc": { 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "enabled": null 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "stretch": { 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "enabled": null 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "network": { 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "provider": null 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "storage": { 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "device-set": { 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:08.025 INFO:tasks.workunit.client.0.smithi033.stdout: "non-portable": null, 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "portable": null, 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "total": null 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "csi": { 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "kubernetes": { 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "node": { 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "kubernetes-total": null, 2024-09-21T09:51:08.026 INFO:tasks.workunit.client.0.smithi033.stdout: "with-ceph-daemons": null, 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-cephfs-plugin": null, 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-nfs-plugin": null, 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-rbd-plugin": null 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "usage": { 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "storage-class": { 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket": null, 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs": null, 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "nfs": null, 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd": null, 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: "total": null 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "services": {}, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "usage": { 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 105, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "pools": 4, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_by_class": { 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "ssd": { 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "total_avail_bytes": 765516980224, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "total_bytes": 767960285184, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_bytes": 2443304960, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_raw_bytes": 2443304960, 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_raw_ratio": 0.003181551117449999 2024-09-21T09:51:08.028 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.029 INFO:tasks.workunit.client.0.smithi033.stdout: "total_avail_bytes": 765516980224, 2024-09-21T09:51:08.029 INFO:tasks.workunit.client.0.smithi033.stdout: "total_bytes": 767960285184, 2024-09-21T09:51:08.029 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_bytes": 2443304960 2024-09-21T09:51:08.029 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.029 INFO:tasks.workunit.client.0.smithi033.stdout:} 2024-09-21T09:51:08.029 INFO:tasks.workunit.client.0.smithi033.stdout:Device report is generated separately. To see it run 'ceph telemetry preview-device'. 2024-09-21T09:51:08.029 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph telemetry preview-device 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE Bus STOPPED 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE Bus STARTING 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE Serving on http://:::9283 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE Bus STARTED 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:07] ENGINE Bus STOPPING 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2577998130' entity='client.admin' 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.030 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1547245403' entity='client.admin' 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/622983401' entity='client.admin' 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2577998130' entity='client.admin' 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1547245403' entity='client.admin' 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/622983401' entity='client.admin' 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.032 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:07 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2577998130' entity='client.admin' 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1547245403' entity='client.admin' 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/622983401' entity='client.admin' 2024-09-21T09:51:08.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:08.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:08.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:07 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout:{ 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout: "069abda2-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout: "INTEL_SSDPEDMD400G4_069b3e9e-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout: "20240921-094500": { 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout: "device": { 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout: "protocol": "NVMe", 2024-09-21T09:51:08.471 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "nvme" 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "host_id": "069abda2-77ff-11ef-904e-ac1f6ba5d294", 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "json_format_version": [ 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "local_time": { 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "asctime": "Sat Sep 21 09:44:59 2024 UTC", 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "time_t": 1726911899 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "logical_block_size": 512, 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_controller_id": 0, 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:51:08.472 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_namespaces": [ 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "capacity": { 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "formatted_lba_size": 512, 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 1, 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "size": { 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "utilization": { 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.473 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_pci_vendor": { 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 32902, 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "subsystem_id": 32902 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "Device stats": { 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_fw_downgrades": { 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "crc_error_count": { 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.474 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "erase_fail_count": { 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "host_bytes_written": { 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 39721974 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.475 INFO:tasks.workunit.client.0.smithi033.stdout: "host_ctx_wear_used": { 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 1 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "media_bytes_read": { 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "nand_bytes_written": { 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 63238513 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_stat_indicator": { 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "pll_lock_loss_count": { 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "program_fail_count": { 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "soft_ecc_err_rate": { 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "thermal_throttle_status": { 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "cnt": 0, 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "pct": 0 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_host_reads": { 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:08.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_media_wear": { 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": "63.999023" 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_timer": { 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "unexp_power_loss": { 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "wear_leveling": { 2024-09-21T09:51:08.479 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 86, 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: "avg": 4199, 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: "max": 4221, 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 4176 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:51:08.480 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare": 100, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare_threshold": 10, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "controller_busy_time": 19, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "critical_warning": 0, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_read": 511399621, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_written": 2603219310, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "host_reads": 7827858467, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "host_writes": 43981992044, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "media_errors": 0, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_err_log_entries": 0, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "percentage_used": 14, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycles": 55292, 2024-09-21T09:51:08.481 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_hours": 67712, 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": 25, 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "unsafe_shutdowns": 780 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_vendor": "intel", 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_version": { 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "string": "<1.2", 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycle_count": 55292, 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_time": { 2024-09-21T09:51:08.482 INFO:tasks.workunit.client.0.smithi033.stdout: "hours": 67712 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: "serial_number": "deleted", 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: "smart_status": { 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme": { 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: "passed": true 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl": { 2024-09-21T09:51:08.483 INFO:tasks.workunit.client.0.smithi033.stdout: "argv": [ 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl", 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "-x", 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "--json=o", 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "/dev/nvme0n1" 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "build_info": "(local build)", 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "exit_status": 0, 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "output": [ 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:51:08.484 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "Serial Number: deleted", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller ID: 0", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "NVMe Version: <1.2", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "Number of Namespaces: 1", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:51:08.485 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "Local Time is: Sat Sep 21 09:44:59 2024 UTC", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported Power States", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.486 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 512 0 2", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: " 1 - 512 8 2", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: " 2 - 512 16 2", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: " 3 - 4096 0 0", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: " 4 - 4096 8 0", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: " 5 - 4096 64 0", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: " 6 - 4096 128 0", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:51:08.487 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Critical Warning: 0x00", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Temperature: 25 Celsius", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare: 100%", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Percentage Used: 14%", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Read: 511,399,621 [261 TB]", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Written: 2,603,219,310 [1.33 PB]", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Read Commands: 7,827,858,467", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Write Commands: 43,981,992,044", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller Busy Time: 19", 2024-09-21T09:51:08.488 INFO:tasks.workunit.client.0.smithi033.stdout: "Power Cycles: 55,292", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "Power On Hours: 67,712", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "Unsafe Shutdowns: 780", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information Log Entries: 0", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "No Errors Logged", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "" 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:51:08.489 INFO:tasks.workunit.client.0.smithi033.stdout: "svn_revision": "5155", 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: "version": [ 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": { 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: "current": 25 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: "user_capacity": { 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.490 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: "069c6a6c-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: "INTEL_SSDPEDMD400G4_069cdbaa-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: "20240921-094503": { 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: "device": { 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: "protocol": "NVMe", 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "nvme" 2024-09-21T09:51:08.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "host_id": "069c6a6c-77ff-11ef-904e-ac1f6ba5d294", 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "json_format_version": [ 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "local_time": { 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "asctime": "Sat Sep 21 09:45:03 2024 UTC", 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "time_t": 1726911903 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "logical_block_size": 512, 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:51:08.492 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_controller_id": 0, 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_namespaces": [ 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "capacity": { 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "formatted_lba_size": 512, 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 1, 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "size": { 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.493 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: "utilization": { 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_pci_vendor": { 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 32902, 2024-09-21T09:51:08.494 INFO:tasks.workunit.client.0.smithi033.stdout: "subsystem_id": 32902 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "Device stats": { 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_fw_downgrades": { 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "crc_error_count": { 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.495 INFO:tasks.workunit.client.0.smithi033.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "erase_fail_count": { 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "host_bytes_written": { 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 33753944 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.496 INFO:tasks.workunit.client.0.smithi033.stdout: "host_ctx_wear_used": { 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 1 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "media_bytes_read": { 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "nand_bytes_written": { 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 56784585 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.497 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_stat_indicator": { 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: "pll_lock_loss_count": { 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: "program_fail_count": { 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "soft_ecc_err_rate": { 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.499 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: "thermal_throttle_status": { 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: "cnt": 0, 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: "pct": 0 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_host_reads": { 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.500 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_media_wear": { 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": "63.999023" 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_timer": { 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: "unexp_power_loss": { 2024-09-21T09:51:08.501 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: "wear_leveling": { 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 88, 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: "avg": 3838, 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: "max": 3855, 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 3821 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.502 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare": 100, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare_threshold": 10, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "controller_busy_time": 18, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "critical_warning": 0, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_read": 450935655, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_written": 2212098498, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "host_reads": 7207456630, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "host_writes": 38408967915, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "media_errors": 0, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_err_log_entries": 0, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "percentage_used": 12, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycles": 58006, 2024-09-21T09:51:08.503 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_hours": 58153, 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": 23, 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "unsafe_shutdowns": 1361 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_vendor": "intel", 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_version": { 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "string": "<1.2", 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycle_count": 58006, 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_time": { 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "hours": 58153 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "serial_number": "deleted", 2024-09-21T09:51:08.504 INFO:tasks.workunit.client.0.smithi033.stdout: "smart_status": { 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme": { 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "passed": true 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl": { 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "argv": [ 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl", 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "-x", 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "--json=o", 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "/dev/nvme0n1" 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "build_info": "(local build)", 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "exit_status": 0, 2024-09-21T09:51:08.505 INFO:tasks.workunit.client.0.smithi033.stdout: "output": [ 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "Serial Number: deleted", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller ID: 0", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "NVMe Version: <1.2", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "Number of Namespaces: 1", 2024-09-21T09:51:08.506 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "Local Time is: Sat Sep 21 09:45:03 2024 UTC", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported Power States", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:51:08.507 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 512 0 2", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: " 1 - 512 8 2", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: " 2 - 512 16 2", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: " 3 - 4096 0 0", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: " 4 - 4096 8 0", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: " 5 - 4096 64 0", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: " 6 - 4096 128 0", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.508 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "Critical Warning: 0x00", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "Temperature: 23 Celsius", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare: 100%", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "Percentage Used: 12%", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Read: 450,935,655 [230 TB]", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Written: 2,212,098,498 [1.13 PB]", 2024-09-21T09:51:08.509 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Read Commands: 7,207,456,630", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Write Commands: 38,408,967,915", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller Busy Time: 18", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "Power Cycles: 58,006", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "Power On Hours: 58,153", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "Unsafe Shutdowns: 1,361", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information Log Entries: 0", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "No Errors Logged", 2024-09-21T09:51:08.510 INFO:tasks.workunit.client.0.smithi033.stdout: "" 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: "svn_revision": "5155", 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: "version": [ 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": { 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: "current": 23 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.511 INFO:tasks.workunit.client.0.smithi033.stdout: "user_capacity": { 2024-09-21T09:51:08.512 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.512 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.512 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.512 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.512 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.512 INFO:tasks.workunit.client.0.smithi033.stdout:} 2024-09-21T09:51:08.513 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph telemetry preview-all 2024-09-21T09:51:08.513 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:51:08.513 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:08] ENGINE Bus STOPPED 2024-09-21T09:51:08.513 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:08] ENGINE Bus STARTING 2024-09-21T09:51:08.513 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:08] ENGINE Serving on http://:::9283 2024-09-21T09:51:08.513 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:08] ENGINE Bus STARTED 2024-09-21T09:51:08.920 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:08.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:08.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:08.961 INFO:tasks.workunit.client.0.smithi033.stdout:{ 2024-09-21T09:51:08.961 INFO:tasks.workunit.client.0.smithi033.stdout: "device_report": { 2024-09-21T09:51:08.961 INFO:tasks.workunit.client.0.smithi033.stdout: "069abda2-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: "INTEL_SSDPEDMD400G4_069b3e9e-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: "20240921-094500": { 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: "device": { 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: "protocol": "NVMe", 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "nvme" 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.962 INFO:tasks.workunit.client.0.smithi033.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: "host_id": "069abda2-77ff-11ef-904e-ac1f6ba5d294", 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: "json_format_version": [ 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: "local_time": { 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: "asctime": "Sat Sep 21 09:44:59 2024 UTC", 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: "time_t": 1726911899 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: "logical_block_size": 512, 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:51:08.963 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_controller_id": 0, 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_namespaces": [ 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "capacity": { 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "formatted_lba_size": 512, 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 1, 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "size": { 2024-09-21T09:51:08.964 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: "utilization": { 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_pci_vendor": { 2024-09-21T09:51:08.965 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 32902, 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: "subsystem_id": 32902 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: "Device stats": { 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_fw_downgrades": { 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: "crc_error_count": { 2024-09-21T09:51:08.966 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: "erase_fail_count": { 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.967 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "host_bytes_written": { 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 39721974 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "host_ctx_wear_used": { 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 1 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "media_bytes_read": { 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.968 INFO:tasks.workunit.client.0.smithi033.stdout: "nand_bytes_written": { 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 63238513 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_stat_indicator": { 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "pll_lock_loss_count": { 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "program_fail_count": { 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "soft_ecc_err_rate": { 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "thermal_throttle_status": { 2024-09-21T09:51:08.970 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "cnt": 0, 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "pct": 0 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_host_reads": { 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_media_wear": { 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": "63.999023" 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_timer": { 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.971 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "unexp_power_loss": { 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "wear_leveling": { 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 86, 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "avg": 4199, 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "max": 4221, 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 4176 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.972 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare": 100, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare_threshold": 10, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "controller_busy_time": 19, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "critical_warning": 0, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_read": 511399621, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_written": 2603219310, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "host_reads": 7827858467, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "host_writes": 43981992044, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "media_errors": 0, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_err_log_entries": 0, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "percentage_used": 14, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycles": 55292, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_hours": 67712, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": 25, 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: "unsafe_shutdowns": 780 2024-09-21T09:51:08.973 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_vendor": "intel", 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_version": { 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "string": "<1.2", 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycle_count": 55292, 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_time": { 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "hours": 67712 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "serial_number": "deleted", 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "smart_status": { 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme": { 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: "passed": true 2024-09-21T09:51:08.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl": { 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "argv": [ 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl", 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "-x", 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "--json=o", 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "/dev/nvme0n1" 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "build_info": "(local build)", 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "exit_status": 0, 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "output": [ 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:51:08.975 INFO:tasks.workunit.client.0.smithi033.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Serial Number: deleted", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller ID: 0", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "NVMe Version: <1.2", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Number of Namespaces: 1", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Local Time is: Sat Sep 21 09:44:59 2024 UTC", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:51:08.976 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported Power States", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 512 0 2", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: " 1 - 512 8 2", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: " 2 - 512 16 2", 2024-09-21T09:51:08.977 INFO:tasks.workunit.client.0.smithi033.stdout: " 3 - 4096 0 0", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: " 4 - 4096 8 0", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: " 5 - 4096 64 0", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: " 6 - 4096 128 0", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "Critical Warning: 0x00", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "Temperature: 25 Celsius", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare: 100%", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "Percentage Used: 14%", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Read: 511,399,621 [261 TB]", 2024-09-21T09:51:08.978 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Written: 2,603,219,310 [1.33 PB]", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Read Commands: 7,827,858,467", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Write Commands: 43,981,992,044", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller Busy Time: 19", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Power Cycles: 55,292", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Power On Hours: 67,712", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Unsafe Shutdowns: 780", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information Log Entries: 0", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "No Errors Logged", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "" 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:51:08.979 INFO:tasks.workunit.client.0.smithi033.stdout: "svn_revision": "5155", 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: "version": [ 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": { 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: "current": 25 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: "user_capacity": { 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.980 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "069c6a6c-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "INTEL_SSDPEDMD400G4_069cdbaa-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "20240921-094503": { 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "device": { 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "protocol": "NVMe", 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "nvme" 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:51:08.981 INFO:tasks.workunit.client.0.smithi033.stdout: "host_id": "069c6a6c-77ff-11ef-904e-ac1f6ba5d294", 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: "json_format_version": [ 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: "local_time": { 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: "asctime": "Sat Sep 21 09:45:03 2024 UTC", 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: "time_t": 1726911903 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.982 INFO:tasks.workunit.client.0.smithi033.stdout: "logical_block_size": 512, 2024-09-21T09:51:08.983 INFO:tasks.workunit.client.0.smithi033.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:51:08.983 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_controller_id": 0, 2024-09-21T09:51:08.983 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:51:08.983 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_namespaces": [ 2024-09-21T09:51:08.983 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:08.983 INFO:tasks.workunit.client.0.smithi033.stdout: "capacity": { 2024-09-21T09:51:08.983 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.983 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: "formatted_lba_size": 512, 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 1, 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: "size": { 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: "utilization": { 2024-09-21T09:51:08.984 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_pci_vendor": { 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 32902, 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: "subsystem_id": 32902 2024-09-21T09:51:08.985 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: "Device stats": { 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_fw_downgrades": { 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: "crc_error_count": { 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.986 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.987 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.987 INFO:tasks.workunit.client.0.smithi033.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:51:08.987 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.987 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.987 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.987 INFO:tasks.workunit.client.0.smithi033.stdout: "erase_fail_count": { 2024-09-21T09:51:08.987 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.987 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: "host_bytes_written": { 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 33753944 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: "host_ctx_wear_used": { 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 1 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.988 INFO:tasks.workunit.client.0.smithi033.stdout: "media_bytes_read": { 2024-09-21T09:51:08.989 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.989 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.989 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.989 INFO:tasks.workunit.client.0.smithi033.stdout: "nand_bytes_written": { 2024-09-21T09:51:08.989 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.989 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 56784585 2024-09-21T09:51:08.989 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.989 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_stat_indicator": { 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: "pll_lock_loss_count": { 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: "program_fail_count": { 2024-09-21T09:51:08.990 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.991 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.992 INFO:tasks.workunit.client.0.smithi033.stdout: "soft_ecc_err_rate": { 2024-09-21T09:51:08.992 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.992 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.992 INFO:tasks.workunit.client.0.smithi033.stdout: "thermal_throttle_status": { 2024-09-21T09:51:08.992 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.992 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: "cnt": 0, 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: "pct": 0 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_host_reads": { 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_media_wear": { 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.993 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": "63.999023" 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_timer": { 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: "unexp_power_loss": { 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: "wear_leveling": { 2024-09-21T09:51:08.994 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 88, 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: "avg": 3838, 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: "max": 3855, 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 3821 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare": 100, 2024-09-21T09:51:08.995 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare_threshold": 10, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "controller_busy_time": 18, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "critical_warning": 0, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_read": 450935655, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_written": 2212098498, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "host_reads": 7207456630, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "host_writes": 38408967915, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "media_errors": 0, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "num_err_log_entries": 0, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "percentage_used": 12, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycles": 58006, 2024-09-21T09:51:08.996 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_hours": 58153, 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": 23, 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "unsafe_shutdowns": 1361 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_vendor": "intel", 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_version": { 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "string": "<1.2", 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycle_count": 58006, 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_time": { 2024-09-21T09:51:08.997 INFO:tasks.workunit.client.0.smithi033.stdout: "hours": 58153 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: "serial_number": "deleted", 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: "smart_status": { 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme": { 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: "passed": true 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl": { 2024-09-21T09:51:08.998 INFO:tasks.workunit.client.0.smithi033.stdout: "argv": [ 2024-09-21T09:51:08.999 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl", 2024-09-21T09:51:08.999 INFO:tasks.workunit.client.0.smithi033.stdout: "-x", 2024-09-21T09:51:08.999 INFO:tasks.workunit.client.0.smithi033.stdout: "--json=o", 2024-09-21T09:51:08.999 INFO:tasks.workunit.client.0.smithi033.stdout: "/dev/nvme0n1" 2024-09-21T09:51:08.999 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:08.999 INFO:tasks.workunit.client.0.smithi033.stdout: "build_info": "(local build)", 2024-09-21T09:51:08.999 INFO:tasks.workunit.client.0.smithi033.stdout: "exit_status": 0, 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "output": [ 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "Serial Number: deleted", 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:51:09.000 INFO:tasks.workunit.client.0.smithi033.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:51:09.001 INFO:tasks.workunit.client.0.smithi033.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:51:09.001 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller ID: 0", 2024-09-21T09:51:09.001 INFO:tasks.workunit.client.0.smithi033.stdout: "NVMe Version: <1.2", 2024-09-21T09:51:09.001 INFO:tasks.workunit.client.0.smithi033.stdout: "Number of Namespaces: 1", 2024-09-21T09:51:09.001 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:51:09.001 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:51:09.001 INFO:tasks.workunit.client.0.smithi033.stdout: "Local Time is: Sat Sep 21 09:45:03 2024 UTC", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported Power States", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 512 0 2", 2024-09-21T09:51:09.002 INFO:tasks.workunit.client.0.smithi033.stdout: " 1 - 512 8 2", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: " 2 - 512 16 2", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: " 3 - 4096 0 0", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: " 4 - 4096 8 0", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: " 5 - 4096 64 0", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: " 6 - 4096 128 0", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "Critical Warning: 0x00", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "Temperature: 23 Celsius", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare: 100%", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:51:09.003 INFO:tasks.workunit.client.0.smithi033.stdout: "Percentage Used: 12%", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Read: 450,935,655 [230 TB]", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Written: 2,212,098,498 [1.13 PB]", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Read Commands: 7,207,456,630", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Write Commands: 38,408,967,915", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller Busy Time: 18", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Power Cycles: 58,006", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Power On Hours: 58,153", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Unsafe Shutdowns: 1,361", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information Log Entries: 0", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "No Errors Logged", 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: "" 2024-09-21T09:51:09.004 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: "svn_revision": "5155", 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: "version": [ 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": { 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: "current": 23 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: "user_capacity": { 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:09.005 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "report": { 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "balancer": { 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "active": true, 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "mode": "" 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "channels": [ 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "basic", 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "crash", 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "device", 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: "ident" 2024-09-21T09:51:09.006 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "channels_available": [ 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "basic", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "ident", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "crash", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "device", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "perf" 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "collections_available": [ 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_base", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "device_base", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "crash_base", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "ident_base", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_perf", 2024-09-21T09:51:09.007 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_mds_metadata", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_usage", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_usage_by_class", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_rook_v01", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_memory_metrics", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_options_bluestore" 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "collections_opted_in": [ 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_base", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "device_base", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "crash_base", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "ident_base", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_perf", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_mds_metadata", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_usage", 2024-09-21T09:51:09.008 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_usage_by_class", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_rook_v01", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_memory_metrics", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_options_bluestore" 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "config": { 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "active_changed": [ 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_debug", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_service_ticket_ttl", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bdev_debug_aio", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "client_check_pool_perm", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "client_mount_gid", 2024-09-21T09:51:09.009 INFO:tasks.workunit.client.0.smithi033.stdout: "client_mount_uid", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "client_quota", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "container_image", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "daemonize", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mgr", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mon", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_ms", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_osd", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_paxos", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "keyring", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_block_size", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_cache_size", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_compression", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_log", 2024-09-21T09:51:09.010 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_write_buffer_size", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_file", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_journald", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_stderr", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_allow_pool_delete", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_clock_drift_allowed", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_file", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_journald", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_stderr", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_data_avail_warn", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_election_default_strategy", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_host", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_max_pg_per_osd", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:51:09.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_bug", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_old_message", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "no_config_file", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_default_list", 2024-09-21T09:51:09.012 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_load_list", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_op_order", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_shutdown", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_profile", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-21T09:51:09.013 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_lim", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_res", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target_autotune", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue_cut_off", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recover_clone_overlap", 2024-09-21T09:51:09.014 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recovery_max_chunk", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_load_threshold", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_max_interval", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "public_addr", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "public_addrv", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "public_network", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd_default_features", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd_qos_exclude_ops", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "setgroup", 2024-09-21T09:51:09.015 INFO:tasks.workunit.client.0.smithi033.stdout: "setuser" 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster_changed": [ 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_debug", 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_service_ticket_ttl", 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bdev_debug_aio", 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: "container_image", 2024-09-21T09:51:09.016 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mgr", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mon", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_ms", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_osd", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_paxos", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_file", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_journald", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_stderr", 2024-09-21T09:51:09.017 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_join_fs", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/allow_ptrace", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/container_init", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/migration_current", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/use_repo_digest", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/ssl_server_port", 2024-09-21T09:51:09.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/orchestrator/orchestrator", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/channel_ident", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/description", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/organization", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_allow_pool_delete", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_clock_drift_allowed", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_file", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_data_avail_warn", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_election_default_strategy", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_max_pg_per_osd", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:51:09.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_bug", 2024-09-21T09:51:09.020 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_old_message", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_default_list", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_load_list", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_op_order", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_shutdown", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_profile", 2024-09-21T09:51:09.021 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target_autotune", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue_cut_off", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recover_clone_overlap", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recovery_max_chunk", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_load_threshold", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_max_interval", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:51:09.022 INFO:tasks.workunit.client.0.smithi033.stdout: "public_network", 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_cache_enabled", 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_enable_ops_log", 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_enable_usage_log" 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "contact": null, 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "crashes": [], 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "created": "2024-09-21T09:41:48.442447Z", 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "crush": { 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_algs": { 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: "straw2": 3 2024-09-21T09:51:09.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_sizes": { 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: "2": 1, 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: "4": 2 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_types": { 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: "1": 2, 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: "11": 1 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: "compat_weight_set": false, 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: "device_classes": [ 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: 8 2024-09-21T09:51:09.024 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "num_buckets": 6, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "num_devices": 8, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "num_rules": 1, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "num_types": 12, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "num_weight_sets": 0, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "tunables": { 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "allowed_bucket_algs": 54, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_local_fallback_tries": 0, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_local_tries": 0, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_total_tries": 50, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_descend_once": 1, 2024-09-21T09:51:09.025 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_stable": 1, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_vary_r": 1, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v2_rules": 0, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v3_rules": 0, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v4_buckets": 1, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v5_rules": 0, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "legacy_tunables": 0, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "minimum_required_version": "jewel", 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "optimal_tunables": 1, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "profile": "jewel", 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables": 1, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables2": 1, 2024-09-21T09:51:09.026 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables3": 1, 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables5": 1, 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: "straw_calc_version": 1 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: "description": "upgrade test cluster", 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: "fs": { 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: "feature_flags": { 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: "enable_multiple": true, 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: "ever_enabled_multiple": true 2024-09-21T09:51:09.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "filesystems": [ 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "approx_ctime": "2024-09", 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "balancer_enabled": false, 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_caps": 0, 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_dns": 10, 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_inos": 13, 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_subtrees": 2, 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "ever_allowed_features": 0, 2024-09-21T09:51:09.028 INFO:tasks.workunit.client.0.smithi033.stdout: "explicitly_allowed_features": 0, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "files": 0, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "max_mds": 1, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_data_pools": 1, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_in": 1, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_mds": 1, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": 0, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_standby_replay": 0, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_up": 1, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "snaps": 0, 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: "standby_count_wanted": 0 2024-09-21T09:51:09.029 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_standby_mds": 1, 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: "total_num_mds": 2 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: "hosts": { 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num": 2, 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mds": 2, 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mgr": 2, 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mon": 2, 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_osd": 2 2024-09-21T09:51:09.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: "leaderboard": false, 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: "license": "sharing-1-0", 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: "metadata": { 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 2 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:51:09.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 1, 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 2 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 2 2024-09-21T09:51:09.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.033 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:09.033 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 2 2024-09-21T09:51:09.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.033 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:09.033 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 2 2024-09-21T09:51:09.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.033 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:09.033 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 2 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 3 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:09.034 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2, 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 3 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 3 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 3 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 3 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:09.035 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 3 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 8 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 4, 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 4 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 8 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.036 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 8 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 8 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 8 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 8 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_objectstore": { 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": 8 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "rotational": { 2024-09-21T09:51:09.037 INFO:tasks.workunit.client.0.smithi033.stdout: "0": 8 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "features": { 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "optional": [], 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "persistent": [ 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "kraken", 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "luminous", 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "mimic", 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap-prune", 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "nautilus", 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "octopus", 2024-09-21T09:51:09.038 INFO:tasks.workunit.client.0.smithi033.stdout: "pacific", 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "elector-pinging", 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "quincy" 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "ipv4_addr_mons": 6, 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "ipv6_addr_mons": 0, 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "min_mon_release": 17, 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "v1_addr_mons": 3, 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "v2_addr_mons": 3 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "organization": "ceph-qa", 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster_network": false, 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 8, 2024-09-21T09:51:09.039 INFO:tasks.workunit.client.0.smithi033.stdout: "require_min_compat_client": "luminous", 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "require_osd_release": "quincy" 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "pools": [ 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr" 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 1, 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 1, 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 1, 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:09.040 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 1781760, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 1781760, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 2, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 126, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 236544, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 590368, 2024-09-21T09:51:09.041 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 590368, 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 1771104, 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 113, 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 1667072 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.042 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 32, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 32, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 2, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 98304, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 98304, 2024-09-21T09:51:09.043 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 22, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 3410, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 3410, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 10230, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 51, 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 19456 2024-09-21T09:51:09.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 64, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 64, 2024-09-21T09:51:09.045 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 3, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 0.0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:09.046 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 0 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:09.047 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd" 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 8, 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 8, 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 4, 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:09.048 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 12288, 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 12288, 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 2, 2024-09-21T09:51:09.049 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 1.689964257423071e-08, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 19, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 19, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 57, 2024-09-21T09:51:09.050 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 2, 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 2048 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd": { 2024-09-21T09:51:09.051 INFO:tasks.workunit.client.0.smithi033.stdout: "mirroring_by_pool": [ 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: false 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: "num_images_by_pool": [ 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pools": 1 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: "report_id": "baa922a1-6d4a-42b6-88d5-8a719a132b6b", 2024-09-21T09:51:09.052 INFO:tasks.workunit.client.0.smithi033.stdout: "report_timestamp": "2024-09-21T09:51:08.943999", 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "report_version": 1, 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "rook": { 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster": { 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "external-mode": null, 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "allow-multiple-per-node": null, 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "count": null, 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "max-id": null, 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "pvc": { 2024-09-21T09:51:09.053 INFO:tasks.workunit.client.0.smithi033.stdout: "enabled": null 2024-09-21T09:51:09.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.054 INFO:tasks.workunit.client.0.smithi033.stdout: "stretch": { 2024-09-21T09:51:09.054 INFO:tasks.workunit.client.0.smithi033.stdout: "enabled": null 2024-09-21T09:51:09.054 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.054 INFO:tasks.workunit.client.0.smithi033.stdout: "network": { 2024-09-21T09:51:09.054 INFO:tasks.workunit.client.0.smithi033.stdout: "provider": null 2024-09-21T09:51:09.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: "storage": { 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: "device-set": { 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: "non-portable": null, 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: "portable": null, 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: "total": null 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.055 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: "csi": { 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: "kubernetes": { 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: "node": { 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:09.056 INFO:tasks.workunit.client.0.smithi033.stdout: "kubernetes-total": null, 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: "with-ceph-daemons": null, 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-cephfs-plugin": null, 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-nfs-plugin": null, 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-rbd-plugin": null 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: "usage": { 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: "storage-class": { 2024-09-21T09:51:09.057 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket": null, 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs": null, 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: "nfs": null, 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd": null, 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: "total": null 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.058 INFO:tasks.workunit.client.0.smithi033.stdout: "services": {}, 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "usage": { 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 105, 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "pools": 4, 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_by_class": { 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "ssd": { 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "total_avail_bytes": 765516980224, 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "total_bytes": 767960285184, 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_bytes": 2443304960, 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_raw_bytes": 2443304960, 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_raw_ratio": 0.003181551117449999 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:09.060 INFO:tasks.workunit.client.0.smithi033.stdout: "total_avail_bytes": 765516980224, 2024-09-21T09:51:09.060 INFO:tasks.workunit.client.0.smithi033.stdout: "total_bytes": 767960285184, 2024-09-21T09:51:09.060 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_bytes": 2443304960 2024-09-21T09:51:09.060 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.060 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:09.060 INFO:tasks.workunit.client.0.smithi033.stdout:} 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: pgmap v89: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:09.061 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:08 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:09.062 INFO:tasks.workunit.client.0.smithi033.stderr:++ ceph telemetry collection ls 2024-09-21T09:51:09.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:08] "GET /metrics HTTP/1.1" 200 236481 "" "Prometheus/2.43.0" 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:+ COLLECTIONS='New collections are available: 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:['\''basic_base'\'', '\''basic_mds_metadata'\'', '\''basic_pool_options_bluestore'\'', '\''basic_pool_usage'\'', '\''basic_rook_v01'\'', '\''basic_usage_by_class'\'', '\''crash_base'\'', '\''device_base'\'', '\''ident_base'\'', '\''perf_memory_metrics'\'', '\''perf_perf'\''] 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:NAME STATUS DESC 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:51:09.424 INFO:tasks.workunit.client.0.smithi033.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster ' 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:+ NEW_COLLECTIONS=("perf_perf" "basic_mds_metadata" "basic_pool_usage" "basic_rook_v01" "perf_memory_metrics" "basic_pool_options_bluestore") 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:+ [[ New collections are available: 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:NAME STATUS DESC 2024-09-21T09:51:09.425 INFO:tasks.workunit.client.0.smithi033.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_perf* ]] 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:+ [[ New collections are available: 2024-09-21T09:51:09.426 INFO:tasks.workunit.client.0.smithi033.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:NAME STATUS DESC 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:51:09.427 INFO:tasks.workunit.client.0.smithi033.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_mds_metadata* ]] 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:+ [[ New collections are available: 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:NAME STATUS DESC 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:51:09.428 INFO:tasks.workunit.client.0.smithi033.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_usage* ]] 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:+ [[ New collections are available: 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:NAME STATUS DESC 2024-09-21T09:51:09.429 INFO:tasks.workunit.client.0.smithi033.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_rook_v01* ]] 2024-09-21T09:51:09.430 INFO:tasks.workunit.client.0.smithi033.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:+ [[ New collections are available: 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:NAME STATUS DESC 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:51:09.431 INFO:tasks.workunit.client.0.smithi033.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *perf_memory_metrics* ]] 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:+ for col in ${NEW_COLLECTIONS[@]} 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:+ [[ New collections are available: 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:['basic_base', 'basic_mds_metadata', 'basic_pool_options_bluestore', 'basic_pool_usage', 'basic_rook_v01', 'basic_usage_by_class', 'crash_base', 'device_base', 'ident_base', 'perf_memory_metrics', 'perf_perf'] 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:Run `ceph telemetry on` to opt-in to these collections. 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr: 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:NAME STATUS DESC 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:basic_base NOT REPORTING: NOT OPTED-IN Basic information about the cluster (capacity, number and type of daemons, version, etc.) 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:basic_mds_metadata NOT REPORTING: NOT OPTED-IN MDS metadata 2024-09-21T09:51:09.432 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_options_bluestore NOT REPORTING: NOT OPTED-IN Per-pool bluestore config options 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:basic_pool_usage NOT REPORTING: NOT OPTED-IN Default pool application and usage statistics 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:basic_rook_v01 NOT REPORTING: NOT OPTED-IN Basic Rook deployment data 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:basic_usage_by_class NOT REPORTING: NOT OPTED-IN Default device class usage statistics 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:crash_base NOT REPORTING: NOT OPTED-IN Information about daemon crashes (daemon type and version, backtrace, etc.) 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:device_base NOT REPORTING: NOT OPTED-IN Information about device health metrics 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:ident_base NOT REPORTING: NOT OPTED-IN User-provided identifying information about the cluster 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:perf_memory_metrics NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Heap stats and mempools for mon and mds 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:perf_perf NOT REPORTING: NOT OPTED-IN, CHANNEL perf IS OFF Information about performance counters of the cluster == *basic_pool_options_bluestore* ]] 2024-09-21T09:51:09.433 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph telemetry on --license sharing-1-0 2024-09-21T09:51:09.896 INFO:tasks.workunit.client.0.smithi033.stdout:Telemetry is on. 2024-09-21T09:51:09.896 INFO:tasks.workunit.client.0.smithi033.stdout:Some channels are disabled, please enable with: 2024-09-21T09:51:09.896 INFO:tasks.workunit.client.0.smithi033.stdout:`ceph telemetry enable channel perf` 2024-09-21T09:51:09.910 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph telemetry enable channel perf 2024-09-21T09:51:09.926 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:09] ENGINE Bus STOPPING 2024-09-21T09:51:09.926 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:51:09.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[27969]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[27969]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[27969]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[27969]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[39045]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[39045]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[39045]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[39045]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.928 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:09.928 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:09 smithi105 ceph-mon[31286]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:09 smithi105 ceph-mon[31286]: from='client.24725 -' entity='client.admin' cmd=[{"prefix": "telemetry preview", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:09 smithi105 ceph-mon[31286]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:09 smithi105 ceph-mon[31286]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:09 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:09 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:09 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:09 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.221 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:09] ENGINE Bus STOPPED 2024-09-21T09:51:10.221 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:09] ENGINE Bus STARTING 2024-09-21T09:51:10.221 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:10] ENGINE Serving on http://:::9283 2024-09-21T09:51:10.221 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:10] ENGINE Bus STARTED 2024-09-21T09:51:10.221 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:10] ENGINE Bus STOPPING 2024-09-21T09:51:10.375 INFO:tasks.workunit.client.0.smithi033.stdout:channel_perf is enabled 2024-09-21T09:51:10.386 INFO:tasks.workunit.client.0.smithi033.stderr:++ ceph config get mgr mgr/telemetry/last_opt_revision 2024-09-21T09:51:10.746 INFO:tasks.workunit.client.0.smithi033.stderr:+ LAST_OPT_REVISION=1 2024-09-21T09:51:10.746 INFO:tasks.workunit.client.0.smithi033.stderr:+ '[' 1 -ne 1 ']' 2024-09-21T09:51:10.746 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph telemetry show 2024-09-21T09:51:10.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:10] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:51:10.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:10] ENGINE Bus STOPPED 2024-09-21T09:51:10.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:10] ENGINE Bus STARTING 2024-09-21T09:51:10.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:10] ENGINE Serving on http://:::9283 2024-09-21T09:51:10.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:51:10] ENGINE Bus STARTED 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='client.24755 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:10.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='client.24755 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2307183428' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.003 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:10 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2307183428' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:51:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "telemetry preview-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: pgmap v90: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='client.14844 -' entity='client.admin' cmd=[{"prefix": "telemetry collection ls", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: mgrmap e22: y(active, since 2m), standbys: x 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='client.24755 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:11.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:11.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:11.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:11.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:11.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:10 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2307183428' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout:{ 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "balancer": { 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "active": true, 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "mode": "" 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "channels": [ 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "basic", 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "crash", 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "device", 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "ident", 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: "perf" 2024-09-21T09:51:11.400 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "channels_available": [ 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "basic", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "ident", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "crash", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "device", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "perf" 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "collections_available": [ 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_base", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "device_base", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "crash_base", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "ident_base", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_perf", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_mds_metadata", 2024-09-21T09:51:11.401 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_usage", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_usage_by_class", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_rook_v01", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_memory_metrics", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_options_bluestore" 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "collections_opted_in": [ 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_base", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "device_base", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "crash_base", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "ident_base", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_perf", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_mds_metadata", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_usage", 2024-09-21T09:51:11.402 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_usage_by_class", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_rook_v01", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_memory_metrics", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_options_bluestore" 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "config": { 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "active_changed": [ 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_debug", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_service_ticket_ttl", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "bdev_debug_aio", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "client_check_pool_perm", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "client_mount_gid", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "client_mount_uid", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "client_quota", 2024-09-21T09:51:11.403 INFO:tasks.workunit.client.0.smithi033.stdout: "container_image", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "daemonize", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mgr", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mon", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_ms", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_osd", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_paxos", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "keyring", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_block_size", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_cache_size", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_compression", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_log", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_write_buffer_size", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_file", 2024-09-21T09:51:11.404 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_journald", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_stderr", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_allow_pool_delete", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_clock_drift_allowed", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_file", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_journald", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_stderr", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_data_avail_warn", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_election_default_strategy", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_host", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_max_pg_per_osd", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:51:11.405 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_bug", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_old_message", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "no_config_file", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_default_list", 2024-09-21T09:51:11.406 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_load_list", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_op_order", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_shutdown", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_profile", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-21T09:51:11.407 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_lim", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_res", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target_autotune", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue_cut_off", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recover_clone_overlap", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recovery_max_chunk", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_load_threshold", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_max_interval", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "public_addr", 2024-09-21T09:51:11.408 INFO:tasks.workunit.client.0.smithi033.stdout: "public_addrv", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "public_network", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd_default_features", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd_qos_exclude_ops", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "setgroup", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "setuser" 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster_changed": [ 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_debug", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_service_ticket_ttl", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "bdev_debug_aio", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "container_image", 2024-09-21T09:51:11.409 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mgr", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mon", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_ms", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_osd", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_paxos", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_file", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_journald", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_stderr", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_join_fs", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/allow_ptrace", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/container_init", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/migration_current", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/use_repo_digest", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-21T09:51:11.410 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/ssl_server_port", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/orchestrator/orchestrator", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/channel_ident", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/channel_perf", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/description", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/enabled", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/organization", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_allow_pool_delete", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_clock_drift_allowed", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_file", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_data_avail_warn", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_election_default_strategy", 2024-09-21T09:51:11.411 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_max_pg_per_osd", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_bug", 2024-09-21T09:51:11.412 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_old_message", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_default_list", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_load_list", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_op_order", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_shutdown", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_profile", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target_autotune", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue_cut_off", 2024-09-21T09:51:11.413 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recover_clone_overlap", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recovery_max_chunk", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_load_threshold", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_max_interval", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "public_network", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_cache_enabled", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_enable_ops_log", 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_enable_usage_log" 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "contact": null, 2024-09-21T09:51:11.414 INFO:tasks.workunit.client.0.smithi033.stdout: "crashes": [], 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "created": "2024-09-21T09:41:48.442447Z", 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "crush": { 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_algs": { 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "straw2": 3 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_sizes": { 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "2": 1, 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "4": 2 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_types": { 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "1": 2, 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "11": 1 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.415 INFO:tasks.workunit.client.0.smithi033.stdout: "compat_weight_set": false, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "device_classes": [ 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: 8 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_buckets": 6, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_devices": 8, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_rules": 1, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_types": 12, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_weight_sets": 0, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "tunables": { 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "allowed_bucket_algs": 54, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_local_fallback_tries": 0, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_local_tries": 0, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_total_tries": 50, 2024-09-21T09:51:11.416 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_descend_once": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_stable": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_vary_r": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v2_rules": 0, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v3_rules": 0, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v4_buckets": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v5_rules": 0, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "legacy_tunables": 0, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "minimum_required_version": "jewel", 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "optimal_tunables": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "profile": "jewel", 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables2": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables3": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables5": 1, 2024-09-21T09:51:11.417 INFO:tasks.workunit.client.0.smithi033.stdout: "straw_calc_version": 1 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "description": "upgrade test cluster", 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "fs": { 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "feature_flags": { 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "enable_multiple": true, 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "ever_enabled_multiple": true 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "filesystems": [ 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "approx_ctime": "2024-09", 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "balancer_enabled": false, 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.418 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_caps": 0, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_dns": 10, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_inos": 13, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_subtrees": 2, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "ever_allowed_features": 0, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "explicitly_allowed_features": 0, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "files": 0, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "max_mds": 1, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_data_pools": 1, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_in": 1, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_mds": 1, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": 0, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_standby_replay": 0, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_up": 1, 2024-09-21T09:51:11.419 INFO:tasks.workunit.client.0.smithi033.stdout: "snaps": 0, 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "standby_count_wanted": 0 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_standby_mds": 1, 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "total_num_mds": 2 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_stats": { 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 20635648, 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 486952, 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 548864, 2024-09-21T09:51:11.420 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 442, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 1299272, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 20, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 1124352, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 15103632, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 21184512 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 20135936, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 338536, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 770048, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 0, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 352, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.421 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 860136, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 12, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 861184, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 14684592, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 20135936 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 36003840, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 1315272, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 10477568, 2024-09-21T09:51:11.422 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1130, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 6248344, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 21, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2051584, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 23767200, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 46481408 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 46374912, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 1355048, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 663552, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 4194304, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1730, 2024-09-21T09:51:11.423 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 10663592, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 23, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 3151360, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 27919920, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 50569216 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.d48952f845b12c2ac6ef02c91277f5cb17a1e8a2": { 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 35405824, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 715096, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 10010624, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 978, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.424 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 6120024, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 21, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2477824, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 23471440, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 45416448 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.0": { 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 274087936, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 970968, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 688128, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1405, 2024-09-21T09:51:11.425 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 5630008, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 45, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2219264, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262515184, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 274776064 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.1": { 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272793600, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 652176, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 32768, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1350, 2024-09-21T09:51:11.426 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 5657024, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2194176, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 261537712, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 272826368 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.2": { 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 271892480, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 798400, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 720896, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1266, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.427 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4699864, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2437632, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 261204072, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 272613376 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.3": { 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 271450112, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 780128, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 393216, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1235, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.428 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4138624, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 45, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 1928960, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 261849888, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 271843328 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.4": { 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272932864, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 721440, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 1007616, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1245, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 5044928, 2024-09-21T09:51:11.429 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 47, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2041088, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262372896, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 273940480 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.5": { 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272621568, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 694312, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 1105920, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1282, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.430 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4544408, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 47, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2234112, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262396224, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.6": { 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272941056, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 866040, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 786432, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1273, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4940176, 2024-09-21T09:51:11.431 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2211072, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262171256, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.7": { 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272900096, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 861632, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 827392, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1324, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4902160, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 45, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2297856, 2024-09-21T09:51:11.432 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262085936, 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "hosts": { 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "num": 2, 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mds": 2, 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mgr": 2, 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mon": 2, 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_osd": 2 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "io_rate": { 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_stats_delta": { 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 0, 2024-09-21T09:51:11.433 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "stamp_delta": "12.002421", 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:11.434 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:11.435 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:11.436 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 0 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.437 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "leaderboard": false, 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "license": "sharing-1-0", 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "mempool": { 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.438 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.439 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.441 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.442 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 222471, 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 72 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 50488, 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 208 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8040, 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 137 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.444 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.447 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.448 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.450 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 149074, 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 33 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 856, 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 2 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.451 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.452 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.453 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.454 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.455 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.456 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.457 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.458 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 1137366, 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 352 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 88, 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.459 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.460 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6688, 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 136 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12744, 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1546 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 5312, 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 103 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.461 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.463 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.465 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.466 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 1119958, 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 329 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 352, 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 4 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.467 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6688, 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 136 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12744, 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1546 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 10624, 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 206 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.469 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.d48952f845b12c2ac6ef02c91277f5cb17a1e8a2": { 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.470 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.471 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.472 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.473 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.474 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 1086630, 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 352 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6688, 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 136 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12744, 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1546 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 5312, 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 103 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.477 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.0": { 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.478 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20272, 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 181 2024-09-21T09:51:11.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 784, 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 7 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8688, 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 181 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20272, 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 181 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 5520, 2024-09-21T09:51:11.480 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 69 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 35195, 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 54289, 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9795 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 133960, 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 197 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 11064, 2024-09-21T09:51:11.481 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 707 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 31412, 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 197 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.482 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 386854, 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 84 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2442398, 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 196 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6072, 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 69 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.483 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 521280, 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 45 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 69784, 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 139 2024-09-21T09:51:11.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235888, 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3530 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.485 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.1": { 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.486 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19152, 2024-09-21T09:51:11.487 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 171 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 784, 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 7 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8208, 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 171 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19152, 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 171 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.488 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4720, 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 59 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8192, 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 2 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 48202, 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8828 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 125120, 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 184 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.489 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 9060, 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 536 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 27144, 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 184 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 736, 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:11.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 402121, 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 87 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12096, 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2419742, 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 189 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.491 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4752, 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 54 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 428608, 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 37 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.492 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8528, 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 22 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235120, 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3494 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.493 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.2": { 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.494 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 18816, 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 168 2024-09-21T09:51:11.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 672, 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 6 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8064, 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 168 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 18816, 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 168 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4400, 2024-09-21T09:51:11.496 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 55 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4096, 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 45468, 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8444 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 119680, 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 176 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.497 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8328, 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 524 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 25600, 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 176 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 736, 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.498 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 393929, 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 85 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12096, 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2422288, 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 181 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4224, 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 48 2024-09-21T09:51:11.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 359104, 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 31 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8840, 2024-09-21T09:51:11.500 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 25 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235632, 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3518 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.501 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.3": { 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:11.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19264, 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:11.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 224, 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 2 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8256, 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19264, 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 3920, 2024-09-21T09:51:11.504 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 49 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 46965, 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8725 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 123760, 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 182 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.505 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 9112, 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 540 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 26464, 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 182 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2208, 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 266401, 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 54 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 30637, 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2453215, 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 191 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4752, 2024-09-21T09:51:11.507 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 54 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 428608, 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 37 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.508 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 10400, 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 20 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235888, 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3530 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.509 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.4": { 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.510 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19376, 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 173 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 672, 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 6 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8304, 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 173 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19376, 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 173 2024-09-21T09:51:11.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 3440, 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 43 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4096, 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 48382, 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8923 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.513 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 126480, 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 186 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 9324, 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 542 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 27320, 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 186 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.514 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 736, 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 908797, 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 68 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12096, 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2405354, 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 169 2024-09-21T09:51:11.515 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4840, 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 55 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 451776, 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 39 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.516 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8944, 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 18 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235376, 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3506 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.517 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.5": { 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.518 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19264, 2024-09-21T09:51:11.519 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 336, 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8256, 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19264, 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.520 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 3120, 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 39 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4096, 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 49349, 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9012 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 127840, 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 188 2024-09-21T09:51:11.521 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 9560, 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 544 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 28000, 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 188 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2208, 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 834798, 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 50 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 30637, 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2411890, 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 183 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.523 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 5544, 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 63 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 474944, 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 41 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 13520, 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 26 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235888, 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3530 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.525 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.6": { 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.527 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20048, 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 179 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 336, 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8592, 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 179 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20048, 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 179 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.528 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2720, 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 34 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 18553, 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 4 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 48626, 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9159 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 125120, 2024-09-21T09:51:11.529 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 184 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 10544, 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 697 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 28340, 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 184 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.530 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 799341, 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 42 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2380108, 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 159 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.531 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4928, 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 56 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 451776, 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 39 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 68952, 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 143 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235120, 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3494 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.7": { 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.534 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20160, 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 180 2024-09-21T09:51:11.535 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 1120, 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 10 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8640, 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 180 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20160, 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 180 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2240, 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 28 2024-09-21T09:51:11.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 282624, 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 10 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 52710, 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9696 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 132600, 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 195 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:11.537 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 11348, 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 706 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 30988, 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 195 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:11.538 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 183787, 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 35 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2400375, 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 217 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:11.539 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6248, 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 71 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 532864, 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 46 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.540 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 71032, 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 171 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235120, 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3494 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.541 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "metadata": { 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 2 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.542 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 1, 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 2 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 2 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 2 2024-09-21T09:51:11.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 2 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 2 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 3 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.544 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2, 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 3 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 3 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 3 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 3 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.545 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 3 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 8 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 4, 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 4 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.546 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 8 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 8 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 8 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 8 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 8 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_objectstore": { 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": 8 2024-09-21T09:51:11.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "rotational": { 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "0": 8 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "features": { 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "optional": [], 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "persistent": [ 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "kraken", 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "luminous", 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "mimic", 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap-prune", 2024-09-21T09:51:11.548 INFO:tasks.workunit.client.0.smithi033.stdout: "nautilus", 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "octopus", 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "pacific", 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "elector-pinging", 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "quincy" 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "ipv4_addr_mons": 6, 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "ipv6_addr_mons": 0, 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "min_mon_release": 17, 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "v1_addr_mons": 3, 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "v2_addr_mons": 3 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "organization": "ceph-qa", 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.549 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster_network": false, 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 8, 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "require_min_compat_client": "luminous", 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "require_osd_release": "quincy" 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_perf_histograms": [ 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "axes": [ 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Latency (usec)", 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 100000, 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:11.550 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 99999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[100000, 199999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[200000, 399999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[400000, 799999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[800000, 1599999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[1600000, 3199999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[3200000, 6399999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[6400000, 12799999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[12800000, 25599999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[25600000, 51199999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[51200000, 102399999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[102400000, 204799999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[204800000, 409599999]", 2024-09-21T09:51:11.551 INFO:tasks.workunit.client.0.smithi033.stdout: "[409600000, 819199999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[819200000, 1638399999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[1638400000, 3276799999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[3276800000, 6553599999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[6553600000, 13107199999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[13107200000, 26214399999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[26214400000, 52428799999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[52428800000, 104857599999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[104857600000, 209715199999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[209715200000, 419430399999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[419430400000, 838860799999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:51:11.552 INFO:tasks.workunit.client.0.smithi033.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "[53687091200000, None]" 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Request size (bytes)", 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 512, 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 511]", 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "[512, 1023]", 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "[1024, 2047]", 2024-09-21T09:51:11.553 INFO:tasks.workunit.client.0.smithi033.stdout: "[2048, 4095]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[4096, 8191]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[8192, 16383]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[16384, 32767]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[32768, 65535]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[65536, 131071]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[131072, 262143]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[262144, 524287]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[524288, 1048575]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[1048576, 2097151]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[2097152, 4194303]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[4194304, 8388607]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[8388608, 16777215]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[16777216, 33554431]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[33554432, 67108863]", 2024-09-21T09:51:11.554 INFO:tasks.workunit.client.0.smithi033.stdout: "[67108864, 134217727]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[134217728, 268435455]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[268435456, 536870911]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[536870912, 1073741823]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[1073741824, 2147483647]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[2147483648, 4294967295]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[4294967296, 8589934591]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[8589934592, 17179869183]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[17179869184, 34359738367]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[34359738368, 68719476735]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[68719476736, 137438953471]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[137438953472, 274877906943]", 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "[274877906944, None]" 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.555 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "osds": [ 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 0, 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.556 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.557 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 1, 2024-09-21T09:51:11.558 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.559 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.560 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 2, 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.561 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.562 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 3, 2024-09-21T09:51:11.563 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.564 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.565 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 4, 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.566 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.567 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.568 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 5, 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.569 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.570 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 6, 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.571 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.572 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.573 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 7, 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.574 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.575 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.576 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "axes": [ 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Latency (usec)", 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 100000, 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 99999]", 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "[100000, 199999]", 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "[200000, 399999]", 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "[400000, 799999]", 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "[800000, 1599999]", 2024-09-21T09:51:11.577 INFO:tasks.workunit.client.0.smithi033.stdout: "[1600000, 3199999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[3200000, 6399999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[6400000, 12799999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[12800000, 25599999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[25600000, 51199999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[51200000, 102399999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[102400000, 204799999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[204800000, 409599999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[409600000, 819199999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[819200000, 1638399999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[1638400000, 3276799999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[3276800000, 6553599999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[6553600000, 13107199999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[13107200000, 26214399999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[26214400000, 52428799999]", 2024-09-21T09:51:11.578 INFO:tasks.workunit.client.0.smithi033.stdout: "[52428800000, 104857599999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[104857600000, 209715199999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[209715200000, 419430399999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[419430400000, 838860799999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "[53687091200000, None]" 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.579 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Request size (bytes)", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 512, 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 511]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[512, 1023]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[1024, 2047]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[2048, 4095]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[4096, 8191]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[8192, 16383]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[16384, 32767]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[32768, 65535]", 2024-09-21T09:51:11.580 INFO:tasks.workunit.client.0.smithi033.stdout: "[65536, 131071]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[131072, 262143]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[262144, 524287]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[524288, 1048575]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[1048576, 2097151]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[2097152, 4194303]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[4194304, 8388607]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[8388608, 16777215]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[16777216, 33554431]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[33554432, 67108863]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[67108864, 134217727]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[134217728, 268435455]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[268435456, 536870911]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[536870912, 1073741823]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[1073741824, 2147483647]", 2024-09-21T09:51:11.581 INFO:tasks.workunit.client.0.smithi033.stdout: "[2147483648, 4294967295]", 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "[4294967296, 8589934591]", 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "[8589934592, 17179869183]", 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "[17179869184, 34359738367]", 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "[34359738368, 68719476735]", 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "[68719476736, 137438953471]", 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "[137438953472, 274877906943]", 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "[274877906944, None]" 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: "osds": [ 2024-09-21T09:51:11.582 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 0, 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.583 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.584 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 1, 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.585 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.586 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.587 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 2, 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.588 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.589 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 3, 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.590 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.591 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.592 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 4, 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.593 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.594 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 5, 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.595 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.596 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.597 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 6, 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.598 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.599 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 7, 2024-09-21T09:51:11.600 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.601 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.602 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "axes": [ 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Latency (usec)", 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 100000, 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:11.603 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 99999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[100000, 199999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[200000, 399999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[400000, 799999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[800000, 1599999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[1600000, 3199999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[3200000, 6399999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[6400000, 12799999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[12800000, 25599999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[25600000, 51199999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[51200000, 102399999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[102400000, 204799999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[204800000, 409599999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[409600000, 819199999]", 2024-09-21T09:51:11.604 INFO:tasks.workunit.client.0.smithi033.stdout: "[819200000, 1638399999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[1638400000, 3276799999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[3276800000, 6553599999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[6553600000, 13107199999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[13107200000, 26214399999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[26214400000, 52428799999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[52428800000, 104857599999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[104857600000, 209715199999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[209715200000, 419430399999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[419430400000, 838860799999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:51:11.605 INFO:tasks.workunit.client.0.smithi033.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "[53687091200000, None]" 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Request size (bytes)", 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 512, 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 511]", 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "[512, 1023]", 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "[1024, 2047]", 2024-09-21T09:51:11.606 INFO:tasks.workunit.client.0.smithi033.stdout: "[2048, 4095]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[4096, 8191]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[8192, 16383]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[16384, 32767]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[32768, 65535]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[65536, 131071]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[131072, 262143]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[262144, 524287]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[524288, 1048575]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[1048576, 2097151]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[2097152, 4194303]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[4194304, 8388607]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[8388608, 16777215]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[16777216, 33554431]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[33554432, 67108863]", 2024-09-21T09:51:11.607 INFO:tasks.workunit.client.0.smithi033.stdout: "[67108864, 134217727]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[134217728, 268435455]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[268435456, 536870911]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[536870912, 1073741823]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[1073741824, 2147483647]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[2147483648, 4294967295]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[4294967296, 8589934591]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[8589934592, 17179869183]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[17179869184, 34359738367]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[34359738368, 68719476735]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[68719476736, 137438953471]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[137438953472, 274877906943]", 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "[274877906944, None]" 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.608 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "osds": [ 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 0, 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.609 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.610 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 1, 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.611 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.612 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.613 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 2, 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.614 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.615 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 3, 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.616 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.617 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.618 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 4, 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.619 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.620 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 5, 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.621 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.622 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.623 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 6, 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.624 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.625 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 7, 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.626 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.627 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.628 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "axes": [ 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Latency (usec)", 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 100000, 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 99999]", 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "[100000, 199999]", 2024-09-21T09:51:11.629 INFO:tasks.workunit.client.0.smithi033.stdout: "[200000, 399999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[400000, 799999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[800000, 1599999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[1600000, 3199999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[3200000, 6399999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[6400000, 12799999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[12800000, 25599999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[25600000, 51199999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[51200000, 102399999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[102400000, 204799999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[204800000, 409599999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[409600000, 819199999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[819200000, 1638399999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[1638400000, 3276799999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[3276800000, 6553599999]", 2024-09-21T09:51:11.630 INFO:tasks.workunit.client.0.smithi033.stdout: "[6553600000, 13107199999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[13107200000, 26214399999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[26214400000, 52428799999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[52428800000, 104857599999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[104857600000, 209715199999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[209715200000, 419430399999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[419430400000, 838860799999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "[53687091200000, None]" 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.631 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Request size (bytes)", 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 512, 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 511]", 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "[512, 1023]", 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "[1024, 2047]", 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "[2048, 4095]", 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "[4096, 8191]", 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "[8192, 16383]", 2024-09-21T09:51:11.632 INFO:tasks.workunit.client.0.smithi033.stdout: "[16384, 32767]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[32768, 65535]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[65536, 131071]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[131072, 262143]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[262144, 524287]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[524288, 1048575]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[1048576, 2097151]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[2097152, 4194303]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[4194304, 8388607]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[8388608, 16777215]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[16777216, 33554431]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[33554432, 67108863]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[67108864, 134217727]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[134217728, 268435455]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[268435456, 536870911]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[536870912, 1073741823]", 2024-09-21T09:51:11.633 INFO:tasks.workunit.client.0.smithi033.stdout: "[1073741824, 2147483647]", 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "[2147483648, 4294967295]", 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "[4294967296, 8589934591]", 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "[8589934592, 17179869183]", 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "[17179869184, 34359738367]", 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "[34359738368, 68719476735]", 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "[68719476736, 137438953471]", 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "[137438953472, 274877906943]", 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "[274877906944, None]" 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: "osds": [ 2024-09-21T09:51:11.634 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 0, 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.635 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.636 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 1, 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.637 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 8, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 3, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 12, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.638 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.639 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 2, 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.640 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.641 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 3, 2024-09-21T09:51:11.642 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.643 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.644 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 4, 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.645 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.646 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.647 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 5, 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.648 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.649 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 6, 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.650 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.651 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.652 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 7, 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.653 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.654 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_counters": { 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:11.655 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "caps": { 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_flush_ack": { 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_grant": { 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_revoke": { 2024-09-21T09:51:11.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_trunc": { 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "dir_commit": { 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "dir_fetch": { 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "dir_merge": { 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "dir_split": { 2024-09-21T09:51:11.657 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "exported_inodes": { 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "forward": { 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_cap_release": { 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_caps": { 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.658 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_caps_dirty": { 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_inode_file_caps": { 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "imported_inodes": { 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "inodes": { 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "inodes_expired": { 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "inodes_pinned": { 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "inodes_with_caps": { 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "load_cent": { 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "openino_dir_fetch": { 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "process_request_cap_release": { 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.660 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "reply_latency": { 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 147843 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "request": { 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "root_rbytes": { 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "root_rfiles": { 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "root_rsnaps": { 2024-09-21T09:51:11.661 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_reply": { 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "subtrees": { 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_cache": { 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_enqueue_scrub": { 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_exportdir": { 2024-09-21T09:51:11.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_flush": { 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_fragmentdir": { 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_fragstats": { 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_inodestats": { 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "num_recovering_enqueued": { 2024-09-21T09:51:11.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "num_recovering_prioritized": { 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "num_recovering_processing": { 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "num_strays": { 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "num_strays_delayed": { 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "num_strays_enqueuing": { 2024-09-21T09:51:11.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_completed": { 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_started": { 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "strays_created": { 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "strays_enqueued": { 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "strays_migrated": { 2024-09-21T09:51:11.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "strays_reintegrated": { 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_log": { 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "ev": { 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "evadd": { 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "evex": { 2024-09-21T09:51:11.666 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "evexd": { 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "evexg": { 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "evtrm": { 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "jlat": { 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 9318416 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "replayed": { 2024-09-21T09:51:11.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "seg": { 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "segadd": { 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "segex": { 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "segexd": { 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "segexg": { 2024-09-21T09:51:11.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "segtrm": { 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_mem": { 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "cap": { 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "cap+": { 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "cap-": { 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "dir": { 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "dir+": { 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "dir-": { 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "dn": { 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "dn+": { 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:11.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "dn-": { 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "heap": { 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 207068 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "ino": { 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "ino+": { 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "ino-": { 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "rss": { 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 38332 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_server": { 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "cap_acquisition_throttle": { 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "cap_revoke_eviction": { 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_request": { 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_session": { 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 16 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_peer_request": { 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "req_create_latency": { 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "req_getattr_latency": { 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 147843 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.673 INFO:tasks.workunit.client.0.smithi033.stdout: "req_getfilelock_latency": { 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "req_getvxattr_latency": { 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "req_link_latency": { 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookup_latency": { 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.674 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookuphash_latency": { 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookupino_latency": { 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookupname_latency": { 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookupparent_latency": { 2024-09-21T09:51:11.675 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookupsnap_latency": { 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lssnap_latency": { 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "req_mkdir_latency": { 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "req_mknod_latency": { 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "req_mksnap_latency": { 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "req_open_latency": { 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "req_readdir_latency": { 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "req_rename_latency": { 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "req_renamesnap_latency": { 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "req_rmdir_latency": { 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "req_rmsnap_latency": { 2024-09-21T09:51:11.678 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "req_rmxattr_latency": { 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setattr_latency": { 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setdirlayout_latency": { 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.679 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setfilelock_latency": { 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setlayout_latency": { 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setxattr_latency": { 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "req_symlink_latency": { 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.680 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "req_unlink_latency": { 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_sessions": { 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "average_load": { 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "avg_session_uptime": { 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 87 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "mdthresh_evicted": { 2024-09-21T09:51:11.681 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "session_add": { 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "session_count": { 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "session_remove": { 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "sessions_open": { 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "sessions_stale": { 2024-09-21T09:51:11.682 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "total_load": { 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.683 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "purge_queue": { 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executed": { 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executing": { 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executing_high_water": { 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.684 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executing_ops": { 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executing_ops_high_water": { 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_item_in_journal": { 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "election_call": { 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.685 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "election_lose": { 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "election_win": { 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "num_elections": { 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": { 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 11 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "session_add": { 2024-09-21T09:51:11.686 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "session_rm": { 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 52 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "session_trim": { 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "paxos": { 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "accept_timeout": { 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "begin": { 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 846 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.687 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_bytes": { 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 846, 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4065103 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_keys": { 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_latency": { 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 846, 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 121997422 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "collect": { 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.688 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_bytes": { 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 146 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_keys": { 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_latency": { 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_timeout": { 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.689 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_uncommitted": { 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "commit": { 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 847 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_bytes": { 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_keys": { 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_latency": { 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.690 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_ack_timeout": { 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_timeout": { 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn": { 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn_latency": { 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.691 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh": { 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 847 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh_latency": { 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 847, 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1850960838 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "restart": { 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state": { 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_bytes": { 2024-09-21T09:51:11.692 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_keys": { 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "start_leader": { 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "start_peon": { 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state": { 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 847 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.693 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_bytes": { 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 847, 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7971301 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_keys": { 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 847, 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5693 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_latency": { 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 847, 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 283993992 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache": { 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.694 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1020054731 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 43859968 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33595392 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2147483648 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10264576 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.695 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:full": { 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277021456 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 271262 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.696 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.697 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 70834514 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.698 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:inc": { 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277021456 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.699 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 41216 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.700 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71064560 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.701 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:kv": { 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 322961408 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 184680971 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.702 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.703 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71171573 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb": { 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "compact": { 2024-09-21T09:51:11.704 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_completed": { 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lasted": { 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 28310214 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_len": { 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_merge": { 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_running": { 2024-09-21T09:51:11.705 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "get_latency": { 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 36184, 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 82964913 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.706 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_latency": { 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_sync_latency": { 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1843, 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 504995419 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.707 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "election_call": { 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "election_lose": { 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "election_win": { 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_elections": { 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 9 2024-09-21T09:51:11.708 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": { 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "session_add": { 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "session_rm": { 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 186 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "session_trim": { 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "paxos": { 2024-09-21T09:51:11.709 INFO:tasks.workunit.client.0.smithi033.stdout: "accept_timeout": { 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "begin": { 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 972 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_bytes": { 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5114206 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_keys": { 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2916 2024-09-21T09:51:11.710 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_latency": { 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 173997229 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "collect": { 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_bytes": { 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_keys": { 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.711 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_latency": { 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_timeout": { 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_uncommitted": { 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "commit": { 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 972 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_bytes": { 2024-09-21T09:51:11.712 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5044280 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_keys": { 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5530 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_latency": { 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 402255956 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_ack_timeout": { 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_timeout": { 2024-09-21T09:51:11.713 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn": { 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn_latency": { 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5999904 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh": { 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 974 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh_latency": { 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 974, 2024-09-21T09:51:11.714 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2592958617 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "restart": { 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state": { 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_bytes": { 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1050 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_keys": { 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.715 INFO:tasks.workunit.client.0.smithi033.stdout: "start_leader": { 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "start_peon": { 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state": { 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_bytes": { 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_keys": { 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.716 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_latency": { 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache": { 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1020054731 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 47947776 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.717 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42426368 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2147483648 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5521408 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:full": { 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.718 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277022519 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270832 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.719 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.720 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 70833881 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:inc": { 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.721 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277022519 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 38813 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.722 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.723 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71065900 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:kv": { 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 322961408 2024-09-21T09:51:11.724 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 184681679 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.725 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.726 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71170865 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb": { 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "compact": { 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.727 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.728 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_completed": { 2024-09-21T09:51:11.728 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.728 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.728 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lasted": { 2024-09-21T09:51:11.728 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33964203 2024-09-21T09:51:11.728 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.728 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_len": { 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_merge": { 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_running": { 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "get_latency": { 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 43614, 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 171857452 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.730 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:51:11.731 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_latency": { 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_sync_latency": { 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2114, 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 669135547 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.d48952f845b12c2ac6ef02c91277f5cb17a1e8a2": { 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:11.732 INFO:tasks.workunit.client.0.smithi033.stdout: "election_call": { 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "election_lose": { 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "election_win": { 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_elections": { 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": { 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.733 INFO:tasks.workunit.client.0.smithi033.stdout: "session_add": { 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "session_rm": { 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "session_trim": { 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "paxos": { 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "accept_timeout": { 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.734 INFO:tasks.workunit.client.0.smithi033.stdout: "begin": { 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 836 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_bytes": { 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 836, 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4038275 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_keys": { 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_latency": { 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 836, 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 240996161 2024-09-21T09:51:11.735 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "collect": { 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_bytes": { 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_keys": { 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_latency": { 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2999952 2024-09-21T09:51:11.736 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_timeout": { 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_uncommitted": { 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "commit": { 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 837 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_bytes": { 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_keys": { 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.737 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_latency": { 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_ack_timeout": { 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_timeout": { 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn": { 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.738 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn_latency": { 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh": { 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 837 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh_latency": { 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 837, 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1981968369 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "restart": { 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state": { 2024-09-21T09:51:11.739 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_bytes": { 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_keys": { 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "start_leader": { 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "start_peon": { 2024-09-21T09:51:11.740 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state": { 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 837 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_bytes": { 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 837, 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7919743 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_keys": { 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 837, 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5618 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_latency": { 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 837, 2024-09-21T09:51:11.741 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 517991733 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache": { 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1020054731 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42795008 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 32006144 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2147483648 2024-09-21T09:51:11.742 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10788864 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:full": { 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.743 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277021456 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 271262 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.744 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.745 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 70834514 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:inc": { 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.746 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277021456 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 41216 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.747 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.748 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71064560 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:kv": { 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 322961408 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.749 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 184680971 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.750 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.751 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71171573 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb": { 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "compact": { 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_completed": { 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.752 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lasted": { 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 32763828 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_len": { 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_merge": { 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_running": { 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "get_latency": { 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 35626, 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 92285215 2024-09-21T09:51:11.753 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:51:11.754 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_latency": { 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_sync_latency": { 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1794, 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 807289558 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.0": { 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.755 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.756 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2375680 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:11.757 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1581056 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1441792 2024-09-21T09:51:11.758 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.759 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:11.760 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.761 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.762 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.763 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3891200 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.764 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1294336 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 186166 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.765 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 25, 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 55958 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 335, 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71401603 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.766 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 323, 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 9584339 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 335, 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 406733 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 335, 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71808336 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.767 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33694 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8910 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.768 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10649 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 725 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 226 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 110, 2024-09-21T09:51:11.769 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3514004 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 264, 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 274372 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 154, 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3285427 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:11.770 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 39402 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5477539 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 99, 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6472508 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 99, 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 527220255598 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.771 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 99, 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 133043675413 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 51005392880 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 48704 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:11.772 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 399797 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 151770600 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 151121 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 69608566 2024-09-21T09:51:11.773 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 25687679 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 900429 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 59, 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 285208 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 253003984 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.774 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 29516713 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2845196 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1862399738 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.775 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 272023552 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 271237120 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3138477772 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 786432 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.776 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85552758 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.777 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 28672 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.778 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73802122 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:11.779 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641645685 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.780 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.781 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71383867 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:11.782 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 57035172 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.783 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.784 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68793484 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:11.785 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641645685 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.786 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 278698 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.787 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71107297 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.788 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.789 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 604 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:11.790 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45997103 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 7, 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3391356 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15749679 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.791 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.792 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.793 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.794 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 604 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45997103 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:11.795 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 7, 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3391356 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15749679 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.796 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305782784 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.1": { 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.797 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:11.798 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2031616 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.799 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1384448 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1245184 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:11.800 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.801 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:11.802 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.803 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.804 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.805 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3350528 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 700416 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 210064 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.806 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:11.807 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 25, 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 58135 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 287, 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 59665849 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 282, 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7098878 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.808 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 287, 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 365579 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 287, 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 60031428 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 62719 2024-09-21T09:51:11.809 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15591 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:11.810 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13797 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 669 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 232 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 115, 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2623473 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:11.811 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 269, 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 228044 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 154, 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2809408 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 102776 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:11.812 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4983810 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 94, 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7018852 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 93, 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 375555613811 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 94, 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 109497588840 2024-09-21T09:51:11.813 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 353, 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 54681211745 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 353, 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37801 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 379437 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:11.814 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 96649729 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 114438 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45843521 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 18018332 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.815 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 309839 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 53, 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 254818 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 165923839 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 22284785 2024-09-21T09:51:11.816 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3388579 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1862399738 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270073856 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269901824 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.817 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3138477772 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 172032 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.818 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85555470 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.819 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8192 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.820 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73819890 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.821 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641666032 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.822 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.823 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71363520 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.824 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 57036980 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.825 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.826 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68791676 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.827 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641666032 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.828 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 254030 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.829 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71111618 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.830 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 97 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.831 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1640827 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 97, 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 637800589 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 602 2024-09-21T09:51:11.832 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 97, 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 36592411 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 97, 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 132882259 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 18 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 18, 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3160177 2024-09-21T09:51:11.833 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 602 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 18, 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2769359 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 18, 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2293151 2024-09-21T09:51:11.834 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 12, 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13771748 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:11.835 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 12, 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5330101 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 12, 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13128135 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.836 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1640827 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 67, 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 620868664 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 67, 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 28492951 2024-09-21T09:51:11.837 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 67, 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 117460973 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 590368 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:11.838 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305192960 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.2": { 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.839 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:11.840 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1753088 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.841 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1187840 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1048576 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.842 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:11.843 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.844 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.845 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.846 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2875392 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.847 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 688128 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 175858 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.848 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 9, 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 11904 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.849 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 239, 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 60236767 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 228, 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12943737 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 239, 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 418992 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.850 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 239, 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 60655759 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37463 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1597 2024-09-21T09:51:11.851 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4036 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 517 2024-09-21T09:51:11.852 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 205 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 100, 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1225862 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 236, 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 274489 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 136, 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1044162 2024-09-21T09:51:11.853 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 82831 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6118277 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10109099 2024-09-21T09:51:11.854 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 89, 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 373715777747 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 153321542323 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 288, 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 63394314049 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.855 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 288, 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 29551 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 433293 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 135486112 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 96564 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.856 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33801186 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 16648043 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 309660 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 50, 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 287276 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.857 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192527012 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 20043992 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1979043 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:11.858 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1862399738 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269860864 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269049856 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3138477772 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 811008 2024-09-21T09:51:11.859 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85556399 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.860 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.861 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73823057 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.862 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.863 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641672995 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.864 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.865 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71356557 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.866 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 57037599 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.867 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.868 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68791057 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.869 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641672995 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.870 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 242654 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.871 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71116031 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.872 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 31 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.873 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 52019984 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:11.874 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4319689 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37957056 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.875 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:11.876 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.877 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.878 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 52019984 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4319689 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37957056 2024-09-21T09:51:11.879 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305180672 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.880 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.3": { 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.881 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1101824 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:11.882 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 827392 2024-09-21T09:51:11.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 688128 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:11.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.885 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:11.886 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.887 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:11.888 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1863680 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.889 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 704512 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.890 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5348 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.891 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 151, 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 41844588 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 148, 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8633007 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 151, 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 239278 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 151, 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42083866 2024-09-21T09:51:11.892 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 40594 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 864 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.893 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8854 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 335 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 213 2024-09-21T09:51:11.894 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 80, 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 716368 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 139, 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 216867 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 59, 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 399739 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.895 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5555149 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 51, 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3883301 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 48, 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 373991339684 2024-09-21T09:51:11.896 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 51, 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71425295254 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 235, 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 558158 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 235, 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 25768 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:11.897 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 251769 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 134071728 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 66707 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 77496601 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.898 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12983832 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312073 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 42, 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 201223 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 230387327 2024-09-21T09:51:11.899 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15304958 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3576473 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1862399738 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.900 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269090816 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 268550144 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3138477772 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 540672 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:11.901 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85556173 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.902 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73827379 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.904 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641671299 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.905 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.906 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71358253 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.907 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 57037448 2024-09-21T09:51:11.908 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.909 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.910 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68791208 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.911 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641671299 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.912 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6959 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 246349 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.913 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71107073 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.914 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:11.915 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.916 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 735637 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.917 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:11.918 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:11.919 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:11.920 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.921 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 735637 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.922 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305192960 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.4": { 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:11.923 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.924 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1327104 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:11.925 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1019904 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 880640 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:11.926 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.927 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.928 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:11.929 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.930 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2281472 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.931 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 708608 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 8, 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 321307 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.932 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 7, 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 22264 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.933 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 198, 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42148142 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 192, 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7452239 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 198, 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 215856 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 198, 2024-09-21T09:51:11.934 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42363998 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 34342 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4747 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:11.935 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12969 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 376 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:11.936 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 232 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 94, 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3100301 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 158, 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 228182 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 64, 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2413987 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.937 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 106043 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5681038 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 41, 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2890771 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 40, 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 181315760770 2024-09-21T09:51:11.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 41, 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 64593627717 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 270, 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 521458 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 270, 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 31097 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:11.939 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 335359 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 83015799 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85845 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 40190223 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.940 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12794179 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 310375 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 39, 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 214523 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 142055651 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:11.941 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15356206 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1991897 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1861455963 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270139392 2024-09-21T09:51:11.942 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270032896 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3137367449 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 106496 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:11.943 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85498945 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.944 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.945 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73880511 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:11.946 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641242091 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.947 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.948 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71787461 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:11.949 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 56999297 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.950 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.951 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68829359 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.952 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641242091 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.953 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 253659 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.954 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71535930 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.955 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 39 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 22 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5882849 2024-09-21T09:51:11.957 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 728215 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5772701 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.958 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.959 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.960 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 22 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.962 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5882849 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 728215 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5772701 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:11.963 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305201152 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.964 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.5": { 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.965 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1232896 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:11.966 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:11.967 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 929792 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 790528 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:11.968 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:11.969 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:11.970 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:11.971 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.972 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2097152 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.973 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 704512 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 208562 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5311 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 176, 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42276933 2024-09-21T09:51:11.975 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 168, 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 9086784 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 176, 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 170584 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 176, 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42447517 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:11.976 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 31748 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 958 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:11.977 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7256 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 345 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 235 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:11.978 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 85, 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 801632 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 139, 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 179244 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 54, 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 547107 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:11.979 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:11.980 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 75787 2024-09-21T09:51:11.981 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.981 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:11.981 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.981 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3297153 2024-09-21T09:51:11.981 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 42, 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3292403 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 39, 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 309073604821 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 42, 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 66506969968 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:11.982 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 272, 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3561876644 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 272, 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 35436 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 227203 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 103271374 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:11.983 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 84263 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71342577 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13460429 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312078 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:11.984 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 34, 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 161256 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 191637146 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15854454 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1867140 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.985 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1861455963 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270974976 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269991936 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3137367449 2024-09-21T09:51:11.986 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 983040 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.987 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85498787 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.988 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.989 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73880669 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.990 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641240905 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.991 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.992 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71788647 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 56999191 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.993 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.994 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68829465 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.995 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641240905 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.996 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6964 2024-09-21T09:51:11.997 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 254904 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71528907 2024-09-21T09:51:11.998 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:11.999 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 41 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:12.000 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.001 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.002 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:12.003 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.004 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.005 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.006 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 590368 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305192960 2024-09-21T09:51:12.007 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.6": { 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:12.008 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1277952 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.009 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:12.010 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 974848 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 835584 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:12.011 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:12.012 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:12.013 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:12.014 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.015 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2187264 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:12.016 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1286144 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:12.017 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 11, 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 54192 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.018 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42087380 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 173, 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 16315080 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 157888 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:12.019 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42245268 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 27344 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4569 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.020 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8100 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 374 2024-09-21T09:51:12.021 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 240 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 86, 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3204171 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 134, 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 207656 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 48, 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2946894 2024-09-21T09:51:12.022 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2795077 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 36, 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2511161 2024-09-21T09:51:12.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 36, 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 386962683653 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 36, 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67818634494 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 459121 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.024 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 31185 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 204826 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 207446838 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 82901 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:12.025 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 89091144 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12025409 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 902093 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 29, 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 125974 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.026 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 311600797 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 19156167 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1921753 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1861455963 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:12.027 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270974976 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270082048 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3137367449 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 892928 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.028 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85498308 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.029 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73872956 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.031 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641237316 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.032 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71792236 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:12.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 56998872 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68829784 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.037 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641237316 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7110 2024-09-21T09:51:12.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7110 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 247433 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.039 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71532711 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.040 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 39 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2804 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 13, 2024-09-21T09:51:12.042 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 64976516 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 15, 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6275495 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 13, 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 30730878 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.043 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.044 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.045 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2804 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 13, 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 64976516 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:12.047 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 15, 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6275495 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 13, 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 30730878 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 590368 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:12.048 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305774592 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.7": { 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.049 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.050 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1257472 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:12.051 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 974848 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 835584 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:12.052 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:12.053 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:12.054 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:12.055 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.056 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.057 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2166784 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1290240 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 145763 2024-09-21T09:51:12.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:12.059 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 11, 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37290 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 51087022 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 181, 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10842897 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:12.060 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 141645 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 51228667 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 32222 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:12.061 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1519 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6757 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.062 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 678 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 271 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 124, 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3240779 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 202, 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 250409 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 78, 2024-09-21T09:51:12.063 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2527664 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15878 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1947884 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 24, 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1829402 2024-09-21T09:51:12.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 24, 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305046592907 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 24, 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 38336156771 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 569025 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33441 2024-09-21T09:51:12.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 227813 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 144886837 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 97203 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:12.066 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 99237150 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 17541818 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 900620 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 23, 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 103860 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:12.067 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 263886316 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 21332868 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4398408 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1861455963 2024-09-21T09:51:12.068 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270974976 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269901824 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3137367449 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1073152 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:12.069 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85481115 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:12.070 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 155648 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:12.071 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 126976 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73619813 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641108365 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:12.074 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71921187 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.075 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 56987410 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.077 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68841246 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:12.078 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641108365 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.079 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 17100 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 260775 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.080 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71645440 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:12.081 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 46 2024-09-21T09:51:12.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 129 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 129, 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 82691313 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 265421 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 131, 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 29586844 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 129, 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 60007948 2024-09-21T09:51:12.084 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 90 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13803239 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 265421 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.085 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12290984 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10123032 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 34 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:12.086 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 34, 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 32526530 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 34, 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14294745 2024-09-21T09:51:12.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 34, 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 30198475 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 36361544 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.088 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 7, 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3001115 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 19686441 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.089 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 590368 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305778688 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.090 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "pools": [ 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr" 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 1, 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 1, 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 1, 2024-09-21T09:51:12.091 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 1781760, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 1781760, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 2, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 2.4504422526661074e-06, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 126, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 236544, 2024-09-21T09:51:12.092 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 590368, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 590368, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 1771104, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 113, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 1667072 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:12.093 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 32, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 32, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 2, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 98304, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 98304, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:12.094 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 22, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 1.3519712638299097e-07, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 3410, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 3410, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 10230, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 51, 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 19456 2024-09-21T09:51:12.095 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 64, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 64, 2024-09-21T09:51:12.096 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 3, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 0.0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:12.097 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 0 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd" 2024-09-21T09:51:12.098 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 8, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 8, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 4, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115966730, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 12288, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:12.099 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 12288, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242371985408, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 2, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 1.689964257423071e-08, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 19, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 19, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 57, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 2, 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 2048 2024-09-21T09:51:12.100 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd": { 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: "mirroring_by_pool": [ 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: false 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: "num_images_by_pool": [ 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pools": 1 2024-09-21T09:51:12.101 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "report_id": "baa922a1-6d4a-42b6-88d5-8a719a132b6b", 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "report_timestamp": "2024-09-21T09:51:11.287248", 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "report_version": 1, 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_stats": { 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "6.15.5" 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "rook": { 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster": { 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "external-mode": null, 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "allow-multiple-per-node": null, 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "count": null, 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "max-id": null, 2024-09-21T09:51:12.102 INFO:tasks.workunit.client.0.smithi033.stdout: "pvc": { 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "enabled": null 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "stretch": { 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "enabled": null 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "network": { 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "provider": null 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "storage": { 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "device-set": { 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "non-portable": null, 2024-09-21T09:51:12.103 INFO:tasks.workunit.client.0.smithi033.stdout: "portable": null, 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "total": null 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "csi": { 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "kubernetes": { 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "node": { 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "kubernetes-total": null, 2024-09-21T09:51:12.104 INFO:tasks.workunit.client.0.smithi033.stdout: "with-ceph-daemons": null, 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-cephfs-plugin": null, 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-nfs-plugin": null, 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-rbd-plugin": null 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "usage": { 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "storage-class": { 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket": null, 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs": null, 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "nfs": null, 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd": null, 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: "total": null 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.105 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: "services": {}, 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_per_pg": [ 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.106 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.066672+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.066672+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.066961+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:12.107 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.108 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3f", 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000", 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.109 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.110 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.111 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.112 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.113 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058702+0000", 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058702+0000", 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.059165+0000", 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.114 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.115 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3e", 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000", 2024-09-21T09:51:12.116 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.117 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.118 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.119 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.120 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060088+0000", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060088+0000", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060359+0000", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:12.121 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.122 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3d", 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000", 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.123 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.124 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.125 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.126 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.127 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.064743+0000", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.064743+0000", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.065047+0000", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:12.128 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.129 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3c", 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000", 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.130 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.131 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.132 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.133 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.134 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.069090+0000", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.069090+0000", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069692+0000", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:12.135 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.136 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3b", 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000", 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.137 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.138 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.139 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.140 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.141 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.053163+0000", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.053163+0000", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.053540+0000", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:12.142 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.143 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3a", 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000", 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.144 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.145 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.146 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.147 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.148 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.774445+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.774445+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.774760+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:12.149 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.150 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.39", 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 31, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000", 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.151 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.152 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.153 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.154 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.155 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.063698+0000", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.063698+0000", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.064091+0000", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:12.156 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.157 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.38", 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000", 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.158 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.159 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.160 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.161 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.162 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.071475+0000", 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.071475+0000", 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.071747+0000", 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.163 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.164 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.37", 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000", 2024-09-21T09:51:12.165 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.166 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.167 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.168 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.169 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057940+0000", 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057940+0000", 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058216+0000", 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.170 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.171 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.36", 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.172 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000", 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.173 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.174 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.175 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.176 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.053264+0000", 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.053264+0000", 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.053543+0000", 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.177 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.178 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.35", 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.179 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000", 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.180 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.181 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.183 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.850604+0000", 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.850604+0000", 2024-09-21T09:51:12.184 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.850968+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.185 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.34", 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.186 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 18, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000", 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.187 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.188 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.190 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.064063+0000", 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.064063+0000", 2024-09-21T09:51:12.191 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.064334+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:12.192 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.33", 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.193 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000", 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.194 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.195 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.197 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058245+0000", 2024-09-21T09:51:12.198 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058245+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058842+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.199 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.32", 2024-09-21T09:51:12.200 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000", 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.201 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.202 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.204 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:12.205 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.051766+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.051766+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.052028+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:12.206 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.207 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.31", 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000", 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.208 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.209 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.211 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:12.212 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.059852+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.059852+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060117+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.213 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.214 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.30", 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000", 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.215 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.216 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.218 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.219 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.480103+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.480103+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.480400+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.220 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.221 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2f", 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 31, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000", 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.222 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.225 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.226 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.056798+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.056798+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.057147+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.227 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.228 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2e", 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000", 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.229 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.232 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.070359+0000", 2024-09-21T09:51:12.233 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.070359+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.070640+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.234 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2d", 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.235 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000", 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.239 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.056360+0000", 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.056360+0000", 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.056659+0000", 2024-09-21T09:51:12.240 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.241 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2c", 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.242 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000", 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.243 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.244 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.246 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.064231+0000", 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.064231+0000", 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.065154+0000", 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:12.247 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.248 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2b", 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.249 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000", 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.250 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.251 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.253 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.062437+0000", 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.062437+0000", 2024-09-21T09:51:12.254 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.062716+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:12.255 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2a", 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.256 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000", 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.257 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.258 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.260 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:12.261 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.051090+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.051090+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.051387+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:12.262 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.29", 2024-09-21T09:51:12.263 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000", 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.264 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.265 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.267 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.268 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057753+0000", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057753+0000", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058079+0000", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:12.269 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.270 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.28", 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000", 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.271 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.272 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.274 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.275 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.056915+0000", 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.056915+0000", 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.057220+0000", 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.276 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.277 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.27", 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000", 2024-09-21T09:51:12.278 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.279 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.281 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.282 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057047+0000", 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057047+0000", 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.057310+0000", 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.283 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.284 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.26", 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000", 2024-09-21T09:51:12.285 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.286 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.288 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.289 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060725+0000", 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060725+0000", 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060976+0000", 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.290 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.291 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.25", 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.292 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000", 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.293 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.294 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.296 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.297 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.052853+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.052853+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.053563+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.298 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.299 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.24", 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000", 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.300 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.301 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.302 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.303 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.304 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.067911+0000", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.067911+0000", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.068204+0000", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:12.305 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.306 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.23", 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000", 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.307 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.308 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.309 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.310 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.311 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058587+0000", 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058587+0000", 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058882+0000", 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.312 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.313 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.22", 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.314 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000", 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.316 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.317 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.318 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060165+0000", 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060165+0000", 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060446+0000", 2024-09-21T09:51:12.319 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.320 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.21", 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.321 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000", 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.325 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.326 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.059270+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.059270+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.059974+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.328 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.20", 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000", 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.329 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.332 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.333 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.048631+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.048631+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.048941+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.335 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1a", 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000", 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.336 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.339 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.340 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.865732+0000", 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.865732+0000", 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.866010+0000", 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.341 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.342 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1b", 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000", 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.343 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.344 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.346 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.347 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.067412+0000", 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.067412+0000", 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.067813+0000", 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.348 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.349 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1b", 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.350 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000", 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.351 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.352 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.354 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.480905+0000", 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.480905+0000", 2024-09-21T09:51:12.355 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.481320+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:12.356 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1a", 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.357 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 35, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:58:23.831301+0000", 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.358 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.359 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.360 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.361 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.859507+0000", 2024-09-21T09:51:12.362 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.859507+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.859829+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:12.363 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.19", 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.364 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000", 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.366 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.367 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.368 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.053295+0000", 2024-09-21T09:51:12.369 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.053295+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.053702+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:12.370 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.371 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.18", 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000", 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.374 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.375 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.376 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.862895+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.862895+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.863181+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:12.377 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.378 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.18", 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000", 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.382 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.383 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.050918+0000", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.050918+0000", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.051248+0000", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:12.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.385 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.19", 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000", 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.386 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.389 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.390 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.865103+0000", 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.865103+0000", 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.865532+0000", 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.392 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.17", 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000", 2024-09-21T09:51:12.393 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.394 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.396 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.397 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.066939+0000", 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.066939+0000", 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.067330+0000", 2024-09-21T09:51:12.398 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.399 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.16", 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.400 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000", 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.401 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.404 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.405 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.860288+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.860288+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.860605+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.406 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.407 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.16", 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000", 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.408 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.409 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.410 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.411 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.412 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057594+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057594+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.057860+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:12.413 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.414 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.17", 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000", 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.415 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.416 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.417 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.418 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.419 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.856719+0000", 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.856719+0000", 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.857026+0000", 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.420 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 3, 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.421 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 3, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.15", 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 27, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000", 2024-09-21T09:51:12.422 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 90, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.424 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 4, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.425 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'3" 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.426 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.059264+0000", 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.059264+0000", 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.059537+0000", 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.427 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.428 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.14", 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.429 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000", 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.432 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.433 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.864160+0000", 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.864160+0000", 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.864446+0000", 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:12.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 12, 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.435 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 12, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.14", 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 36, 2024-09-21T09:51:12.436 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000", 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 2624, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 12, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 4, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 4, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 12, 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 9 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "67'12" 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.440 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.069698+0000", 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.069698+0000", 2024-09-21T09:51:12.441 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.070010+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.442 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.15", 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.443 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000", 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.444 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.445 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.447 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.857951+0000", 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.857951+0000", 2024-09-21T09:51:12.448 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.858242+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:12.449 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.13", 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.450 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000", 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 34, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.451 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.452 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.453 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 6 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.454 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.063116+0000", 2024-09-21T09:51:12.455 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.063116+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.063666+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:12.456 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.12", 2024-09-21T09:51:12.457 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000", 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.458 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.459 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.460 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.461 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.462 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.866875+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.866875+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.867134+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.463 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.464 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.12", 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000", 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.467 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.468 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.469 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.069569+0000", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.069569+0000", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069831+0000", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:12.470 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.471 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.13", 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000", 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.472 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.475 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.476 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.862582+0000", 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.862582+0000", 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862992+0000", 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.478 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.11", 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000", 2024-09-21T09:51:12.479 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.483 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057961+0000", 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057961+0000", 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058311+0000", 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.484 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.485 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.10", 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.486 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000", 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.487 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.490 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.856229+0000", 2024-09-21T09:51:12.491 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.856229+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.856509+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:12.492 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 8, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 8, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.10", 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.493 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 32, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000", 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 12, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 4, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 4, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 4, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 12, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'8" 2024-09-21T09:51:12.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.498 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.068917+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.068917+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069301+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.499 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.500 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.11", 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000", 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.502 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.504 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.505 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861815+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861815+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862118+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:12.506 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.507 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.f", 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000", 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.508 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 22, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 1, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.511 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.512 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.069235+0000", 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.069235+0000", 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069502+0000", 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.513 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.514 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.e", 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000", 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.515 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.516 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.517 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.518 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.519 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.862237+0000", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.862237+0000", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862503+0000", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:12.520 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.521 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.e", 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000", 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.522 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.523 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.524 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.525 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.526 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.068990+0000", 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.068990+0000", 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069273+0000", 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.527 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.528 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.f", 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000", 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.529 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.530 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.531 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.532 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.533 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.857760+0000", 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.857760+0000", 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.858095+0000", 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.534 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.535 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.d", 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000", 2024-09-21T09:51:12.536 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 70, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.537 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.538 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 1, 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1 2024-09-21T09:51:12.539 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.540 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.063070+0000", 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.063070+0000", 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.063342+0000", 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:12.541 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.542 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.c", 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.543 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000", 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.544 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.545 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.546 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.547 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.859443+0000", 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.859443+0000", 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.859702+0000", 2024-09-21T09:51:12.548 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.549 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.c", 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.550 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000", 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.551 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.552 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.553 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.554 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:12.555 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.070074+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.070074+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.070338+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:12.556 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.d", 2024-09-21T09:51:12.557 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000", 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.558 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.559 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.560 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.561 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.562 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.296475+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.296475+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:38.081026+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.563 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.564 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.5", 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:24:15.020915+0000", 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000462138, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.565 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.566 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.567 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.568 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 19, 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.569 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978200+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:46:57.418997+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:46:57.418997+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:46:57.426595+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978200+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:44:57.819716+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:44:57.819716+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 50, 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978200+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978200+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978200+0000", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.570 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:44:57.819716+0000", 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978200+0000", 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978200+0000", 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 79, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 49, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 79, 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.571 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "1.0", 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 58, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000", 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 590368, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 590368, 2024-09-21T09:51:12.572 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:12.573 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 2, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.574 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 126, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 231, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 113, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1628 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.575 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "60'79" 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.576 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.866224+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.866224+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.866500+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.577 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.578 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.3", 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000", 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.579 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.580 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.581 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.582 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.583 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060632+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060632+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.061053+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.584 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.585 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2", 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000", 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.586 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.587 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.588 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.589 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.590 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.298887+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.298887+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.887537+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.591 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.592 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.592 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:12.592 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:12.592 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.592 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.593 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.593 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.593 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:12.593 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.6", 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.594 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:51:16.078381+0000", 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000277308, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.595 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.596 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.597 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.598 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.868800+0000", 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.868800+0000", 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.869192+0000", 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.599 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.600 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.0", 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.601 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000", 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.602 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.603 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.604 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.605 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.056366+0000", 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.056366+0000", 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.056658+0000", 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.606 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.607 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1", 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.608 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000", 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.609 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.610 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.611 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.612 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.677383+0000", 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.677383+0000", 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.861462+0000", 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.613 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:12.614 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.7", 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:37:12.106483+0000", 2024-09-21T09:51:12.615 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000471278, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.616 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.617 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.618 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.619 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861684+0000", 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861684+0000", 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.861899+0000", 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.620 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.621 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1", 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.622 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000", 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.623 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.624 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.625 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.626 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057972+0000", 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057972+0000", 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058235+0000", 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:12.627 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.628 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.0", 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.629 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000", 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.630 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.631 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.632 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.633 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.295964+0000", 2024-09-21T09:51:12.634 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.295964+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.861803+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:12.635 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.4", 2024-09-21T09:51:12.636 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:57:02.417883+0000", 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.001053669, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.637 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.638 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.641 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861254+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861254+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.861547+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.642 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.643 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.2", 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000", 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.644 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.645 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.646 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.647 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.648 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.059929+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.059929+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060191+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:12.649 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.650 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3", 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000", 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.651 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.652 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.653 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.654 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.655 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.677309+0000", 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.677309+0000", 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.861137+0000", 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.656 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:12.657 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.2", 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.658 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:21:29.622677+0000", 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000569086, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 19, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.659 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.660 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 2, 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 2 2024-09-21T09:51:12.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "66'2" 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.662 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861903+0000", 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861903+0000", 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862181+0000", 2024-09-21T09:51:12.663 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.664 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.4", 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.665 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000", 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.666 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.667 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.668 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.669 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.670 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058564+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058564+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058849+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.671 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.672 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.5", 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000", 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.673 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.674 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.675 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.676 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.677 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.677589+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.677589+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.887211+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.678 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.679 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.3", 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 20, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:59:20.123742+0000", 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000403144, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.680 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.681 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.682 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "64'1" 2024-09-21T09:51:12.683 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:12.684 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.866436+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.866436+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.866885+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:12.685 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.686 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.5", 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000", 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.687 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.688 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.689 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.690 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.691 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060083+0000", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060083+0000", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060367+0000", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:12.692 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.693 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.4", 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000", 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.694 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.695 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.696 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.697 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.698 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.298926+0000", 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.298926+0000", 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:38.080316+0000", 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.699 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:12.700 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.0", 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:41:34.343076+0000", 2024-09-21T09:51:12.701 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000557536, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.704 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.705 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.858680+0000", 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.858680+0000", 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.858947+0000", 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:12.706 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.707 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.6", 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:12.708 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000", 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.710 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 1, 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.711 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.712 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060179+0000", 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060179+0000", 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060498+0000", 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.713 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.714 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.7", 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.715 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000", 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.716 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.717 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.718 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.719 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.677625+0000", 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.677625+0000", 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.860425+0000", 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.721 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.1", 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:12.722 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:07:51.936200+0000", 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000698406, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.725 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.726 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.865466+0000", 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.865466+0000", 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.865797+0000", 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.727 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.728 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.7", 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.729 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000", 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.730 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.732 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.733 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058371+0000", 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058371+0000", 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058690+0000", 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.734 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.735 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.6", 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000", 2024-09-21T09:51:12.736 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.737 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.738 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.739 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.740 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.868686+0000", 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.868686+0000", 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.868994+0000", 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.741 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.742 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.8", 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.743 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000", 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.744 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.745 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.746 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.747 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.054785+0000", 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.054785+0000", 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.055068+0000", 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:12.748 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.749 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.9", 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.750 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000", 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.751 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.752 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.753 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.754 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861988+0000", 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861988+0000", 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862674+0000", 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:12.755 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.756 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.9", 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.757 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000", 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.761 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058565+0000", 2024-09-21T09:51:12.762 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058565+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058826+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:12.763 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.8", 2024-09-21T09:51:12.764 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000", 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.765 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.768 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.769 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.860916+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.860916+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.861142+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.770 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.771 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.a", 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000", 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.772 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.773 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.774 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:12.775 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.776 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.481006+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.481006+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.481406+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.777 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.778 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.b", 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 31, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:18:56.780478+0000", 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.779 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.780 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.781 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.782 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.783 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.863790+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.863790+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.865234+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.784 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.785 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.b", 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000", 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.786 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.787 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.788 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 1, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.789 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.790 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.055928+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.055928+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.056216+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.791 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.792 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.a", 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000", 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.793 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.794 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.795 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.797 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.798 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.063293+0000", 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.063293+0000", 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.063533+0000", 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:12.799 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.801 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1d", 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000", 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.802 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.803 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:12.805 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.806 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.864212+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.864212+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.864537+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:12.807 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.808 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1c", 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000", 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.809 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.810 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.811 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.812 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.813 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.773846+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.773846+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.774197+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:12.814 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.815 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1c", 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 18, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000", 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.816 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.817 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.818 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.819 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.820 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.866026+0000", 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.866026+0000", 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.866709+0000", 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.821 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.822 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1d", 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000", 2024-09-21T09:51:12.823 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.824 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.825 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:12.826 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.827 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060536+0000", 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060536+0000", 2024-09-21T09:51:12.828 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060801+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:12.829 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.830 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1f", 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000", 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.831 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.832 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.833 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.834 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.835 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.864185+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.864185+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.864529+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.836 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1e", 2024-09-21T09:51:12.837 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000", 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.839 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.840 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.841 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.842 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.065373+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.065373+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.065677+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.843 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.844 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1e", 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000", 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.847 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:12.848 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:12.849 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.865683+0000", 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.865683+0000", 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.865947+0000", 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.850 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 4, 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:12.851 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 4, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1f", 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 28, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000", 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:12.852 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 570, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.854 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 4, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:12.855 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'4" 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_per_pool": [ 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 24, 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd" 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.856 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 3, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pg": 8, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 8, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 3, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "poolid": 4, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 19, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 2, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 2 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 12288, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 57, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:12.860 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 24 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 192, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pg": 64, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 8, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "poolid": 3, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:12.861 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.862 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.863 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:12.864 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 0, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 192 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 96, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 45, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pg": 32, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 8, 2024-09-21T09:51:12.865 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 45, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "poolid": 2, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 3410, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 66, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 22, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 22, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 14, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.868 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 51, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 19 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 98304, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 10230, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:12.869 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 96 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 3, 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr" 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 79, 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pg": 1, 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 3, 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 79, 2024-09-21T09:51:12.870 INFO:tasks.workunit.client.0.smithi033.stdout: "poolid": 1, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 590368, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 590368, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:12.871 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 2, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:12.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 126, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 231, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 113, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1628 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 1781760, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:12.873 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 1771104, 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 3 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "usage": { 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 105, 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "pools": 4, 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_by_class": { 2024-09-21T09:51:12.874 INFO:tasks.workunit.client.0.smithi033.stdout: "ssd": { 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: "total_avail_bytes": 765516980224, 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: "total_bytes": 767960285184, 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_bytes": 2443304960, 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_raw_bytes": 2443304960, 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_raw_ratio": 0.003181551117449999 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: "total_avail_bytes": 765516980224, 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: "total_bytes": 767960285184, 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_bytes": 2443304960 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout:} 2024-09-21T09:51:12.875 INFO:tasks.workunit.client.0.smithi033.stdout:Device report is generated separately. To see it run 'ceph telemetry show-device'. 2024-09-21T09:51:12.876 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph telemetry show-device 2024-09-21T09:51:12.877 INFO:tasks.workunit.client.0.smithi033.stdout:{ 2024-09-21T09:51:12.877 INFO:tasks.workunit.client.0.smithi033.stdout: "069abda2-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:12.877 INFO:tasks.workunit.client.0.smithi033.stdout: "INTEL_SSDPEDMD400G4_069b3e9e-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:12.877 INFO:tasks.workunit.client.0.smithi033.stdout: "20240921-094500": { 2024-09-21T09:51:12.877 INFO:tasks.workunit.client.0.smithi033.stdout: "device": { 2024-09-21T09:51:12.877 INFO:tasks.workunit.client.0.smithi033.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "protocol": "NVMe", 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "nvme" 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "host_id": "069abda2-77ff-11ef-904e-ac1f6ba5d294", 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "json_format_version": [ 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "local_time": { 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "asctime": "Sat Sep 21 09:44:59 2024 UTC", 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: "time_t": 1726911899 2024-09-21T09:51:12.878 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "logical_block_size": 512, 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_controller_id": 0, 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_namespaces": [ 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "capacity": { 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "formatted_lba_size": 512, 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 1, 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "size": { 2024-09-21T09:51:12.879 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "utilization": { 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_pci_vendor": { 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 32902, 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "subsystem_id": 32902 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.880 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "Device stats": { 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_fw_downgrades": { 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "crc_error_count": { 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "erase_fail_count": { 2024-09-21T09:51:12.881 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "host_bytes_written": { 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 39721974 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "host_ctx_wear_used": { 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 1 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "media_bytes_read": { 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "nand_bytes_written": { 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 63238513 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_stat_indicator": { 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "pll_lock_loss_count": { 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "program_fail_count": { 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.883 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "soft_ecc_err_rate": { 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "thermal_throttle_status": { 2024-09-21T09:51:12.884 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "cnt": 0, 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "pct": 0 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_host_reads": { 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_media_wear": { 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": "63.999023" 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_timer": { 2024-09-21T09:51:12.885 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "unexp_power_loss": { 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "wear_leveling": { 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 86, 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "avg": 4199, 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "max": 4221, 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 4176 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.886 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare": 100, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare_threshold": 10, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "controller_busy_time": 19, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "critical_warning": 0, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_read": 511399621, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_written": 2603219310, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "host_reads": 7827858467, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "host_writes": 43981992044, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "media_errors": 0, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_err_log_entries": 0, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "percentage_used": 14, 2024-09-21T09:51:12.887 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycles": 55292, 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_hours": 67712, 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": 25, 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "unsafe_shutdowns": 780 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_vendor": "intel", 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_version": { 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "string": "<1.2", 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycle_count": 55292, 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_time": { 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "hours": 67712 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "serial_number": "deleted", 2024-09-21T09:51:12.888 INFO:tasks.workunit.client.0.smithi033.stdout: "smart_status": { 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme": { 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "passed": true 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl": { 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "argv": [ 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl", 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "-x", 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "--json=o", 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: "/dev/nvme0n1" 2024-09-21T09:51:12.889 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "build_info": "(local build)", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "exit_status": 0, 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "output": [ 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "Serial Number: deleted", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller ID: 0", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "NVMe Version: <1.2", 2024-09-21T09:51:12.890 INFO:tasks.workunit.client.0.smithi033.stdout: "Number of Namespaces: 1", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Local Time is: Sat Sep 21 09:44:59 2024 UTC", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported Power States", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:51:12.891 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 512 0 2", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: " 1 - 512 8 2", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: " 2 - 512 16 2", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: " 3 - 4096 0 0", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: " 4 - 4096 8 0", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: " 5 - 4096 64 0", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: " 6 - 4096 128 0", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:51:12.892 INFO:tasks.workunit.client.0.smithi033.stdout: "Critical Warning: 0x00", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Temperature: 25 Celsius", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare: 100%", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Percentage Used: 14%", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Read: 511,399,621 [261 TB]", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Written: 2,603,219,310 [1.33 PB]", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Read Commands: 7,827,858,467", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Write Commands: 43,981,992,044", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller Busy Time: 19", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Power Cycles: 55,292", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Power On Hours: 67,712", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Unsafe Shutdowns: 780", 2024-09-21T09:51:12.893 INFO:tasks.workunit.client.0.smithi033.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information Log Entries: 0", 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "No Errors Logged", 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "" 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "svn_revision": "5155", 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "version": [ 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.894 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": { 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "current": 25 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "user_capacity": { 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "069c6a6c-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "INTEL_SSDPEDMD400G4_069cdbaa-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "20240921-094503": { 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "device": { 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:51:12.895 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "protocol": "NVMe", 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "nvme" 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "host_id": "069c6a6c-77ff-11ef-904e-ac1f6ba5d294", 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "json_format_version": [ 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "local_time": { 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "asctime": "Sat Sep 21 09:45:03 2024 UTC", 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "time_t": 1726911903 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "logical_block_size": 512, 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:51:12.896 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_controller_id": 0, 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_namespaces": [ 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "capacity": { 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "formatted_lba_size": 512, 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 1, 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "size": { 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.897 INFO:tasks.workunit.client.0.smithi033.stdout: "utilization": { 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_pci_vendor": { 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 32902, 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "subsystem_id": 32902 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "Device stats": { 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_fw_downgrades": { 2024-09-21T09:51:12.898 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "crc_error_count": { 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "erase_fail_count": { 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.899 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "host_bytes_written": { 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 33753944 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "host_ctx_wear_used": { 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 1 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "media_bytes_read": { 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "nand_bytes_written": { 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.900 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 56784585 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_stat_indicator": { 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "pll_lock_loss_count": { 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "program_fail_count": { 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.901 INFO:tasks.workunit.client.0.smithi033.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "soft_ecc_err_rate": { 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "thermal_throttle_status": { 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "cnt": 0, 2024-09-21T09:51:12.902 INFO:tasks.workunit.client.0.smithi033.stdout: "pct": 0 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_host_reads": { 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_media_wear": { 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": "63.999023" 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_timer": { 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:12.903 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "unexp_power_loss": { 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "wear_leveling": { 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 88, 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "avg": 3838, 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "max": 3855, 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 3821 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:51:12.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare": 100, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare_threshold": 10, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "controller_busy_time": 18, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "critical_warning": 0, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_read": 450935655, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_written": 2212098498, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "host_reads": 7207456630, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "host_writes": 38408967915, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "media_errors": 0, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "num_err_log_entries": 0, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "percentage_used": 12, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycles": 58006, 2024-09-21T09:51:12.905 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_hours": 58153, 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": 23, 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "unsafe_shutdowns": 1361 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_vendor": "intel", 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_version": { 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "string": "<1.2", 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycle_count": 58006, 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_time": { 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "hours": 58153 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "serial_number": "deleted", 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "smart_status": { 2024-09-21T09:51:12.906 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme": { 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "passed": true 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl": { 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "argv": [ 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl", 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "-x", 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "--json=o", 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "/dev/nvme0n1" 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "build_info": "(local build)", 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "exit_status": 0, 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "output": [ 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:51:12.907 INFO:tasks.workunit.client.0.smithi033.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Serial Number: deleted", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller ID: 0", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "NVMe Version: <1.2", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Number of Namespaces: 1", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Local Time is: Sat Sep 21 09:45:03 2024 UTC", 2024-09-21T09:51:12.908 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported Power States", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 512 0 2", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: " 1 - 512 8 2", 2024-09-21T09:51:12.909 INFO:tasks.workunit.client.0.smithi033.stdout: " 2 - 512 16 2", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: " 3 - 4096 0 0", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: " 4 - 4096 8 0", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: " 5 - 4096 64 0", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: " 6 - 4096 128 0", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "Critical Warning: 0x00", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "Temperature: 23 Celsius", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare: 100%", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:51:12.910 INFO:tasks.workunit.client.0.smithi033.stdout: "Percentage Used: 12%", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Read: 450,935,655 [230 TB]", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Written: 2,212,098,498 [1.13 PB]", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Read Commands: 7,207,456,630", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Write Commands: 38,408,967,915", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller Busy Time: 18", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Power Cycles: 58,006", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Power On Hours: 58,153", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Unsafe Shutdowns: 1,361", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information Log Entries: 0", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "No Errors Logged", 2024-09-21T09:51:12.911 INFO:tasks.workunit.client.0.smithi033.stdout: "" 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: "svn_revision": "5155", 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: "version": [ 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": { 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: "current": 23 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: "user_capacity": { 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.912 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.913 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.913 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.913 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.913 INFO:tasks.workunit.client.0.smithi033.stdout:} 2024-09-21T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='client.24755 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.913 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2307183428' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.914 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.915 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.917 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:11 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.918 INFO:tasks.workunit.client.0.smithi033.stderr:+ ceph telemetry show-all 2024-09-21T09:51:12.918 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:51:12.918 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='client.24755 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.918 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.918 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.918 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2307183428' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.919 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.920 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='client.24755 -' entity='client.admin' cmd=[{"prefix": "telemetry on", "license": "sharing-1-0", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='client.14856 -' entity='client.admin' cmd=[{"prefix": "telemetry enable channel", "channels": ["perf"], "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2307183428' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/telemetry/last_opt_revision"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.924 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.925 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:12.926 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:12.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:12.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:11 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:12.932 INFO:tasks.workunit.client.0.smithi033.stdout:{ 2024-09-21T09:51:12.932 INFO:tasks.workunit.client.0.smithi033.stdout: "device_report": { 2024-09-21T09:51:12.932 INFO:tasks.workunit.client.0.smithi033.stdout: "069abda2-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:12.932 INFO:tasks.workunit.client.0.smithi033.stdout: "INTEL_SSDPEDMD400G4_069b3e9e-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:12.932 INFO:tasks.workunit.client.0.smithi033.stdout: "20240921-094500": { 2024-09-21T09:51:12.932 INFO:tasks.workunit.client.0.smithi033.stdout: "device": { 2024-09-21T09:51:12.932 INFO:tasks.workunit.client.0.smithi033.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:51:12.932 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "protocol": "NVMe", 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "nvme" 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "host_id": "069abda2-77ff-11ef-904e-ac1f6ba5d294", 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "json_format_version": [ 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "local_time": { 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "asctime": "Sat Sep 21 09:44:59 2024 UTC", 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "time_t": 1726911899 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "logical_block_size": 512, 2024-09-21T09:51:12.933 INFO:tasks.workunit.client.0.smithi033.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_controller_id": 0, 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_namespaces": [ 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "capacity": { 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "formatted_lba_size": 512, 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 1, 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "size": { 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.934 INFO:tasks.workunit.client.0.smithi033.stdout: "utilization": { 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_pci_vendor": { 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 32902, 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "subsystem_id": 32902 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "Device stats": { 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_fw_downgrades": { 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.935 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "crc_error_count": { 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "erase_fail_count": { 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "host_bytes_written": { 2024-09-21T09:51:12.936 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 39721974 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "host_ctx_wear_used": { 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 1 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "media_bytes_read": { 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "nand_bytes_written": { 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 63238513 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_stat_indicator": { 2024-09-21T09:51:12.937 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "pll_lock_loss_count": { 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "program_fail_count": { 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.938 INFO:tasks.workunit.client.0.smithi033.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "soft_ecc_err_rate": { 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "thermal_throttle_status": { 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "cnt": 0, 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "pct": 0 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.939 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_host_reads": { 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_media_wear": { 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": "63.999023" 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_timer": { 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "unexp_power_loss": { 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.940 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "wear_leveling": { 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 86, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "avg": 4199, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "max": 4221, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 4176 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare": 100, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare_threshold": 10, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "controller_busy_time": 19, 2024-09-21T09:51:12.941 INFO:tasks.workunit.client.0.smithi033.stdout: "critical_warning": 0, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_read": 511399621, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_written": 2603219310, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "host_reads": 7827858467, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "host_writes": 43981992044, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "media_errors": 0, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "num_err_log_entries": 0, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "percentage_used": 14, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycles": 55292, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_hours": 67712, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": 25, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "unsafe_shutdowns": 780 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_vendor": "intel", 2024-09-21T09:51:12.942 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_version": { 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "string": "<1.2", 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycle_count": 55292, 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_time": { 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "hours": 67712 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "serial_number": "deleted", 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "smart_status": { 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme": { 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "passed": true 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.943 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl": { 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "argv": [ 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl", 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "-x", 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "--json=o", 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "/dev/nvme0n1" 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "build_info": "(local build)", 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "exit_status": 0, 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "output": [ 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:51:12.944 INFO:tasks.workunit.client.0.smithi033.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Serial Number: deleted", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller ID: 0", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "NVMe Version: <1.2", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Number of Namespaces: 1", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Local Time is: Sat Sep 21 09:44:59 2024 UTC", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:51:12.945 INFO:tasks.workunit.client.0.smithi033.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported Power States", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 512 0 2", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: " 1 - 512 8 2", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: " 2 - 512 16 2", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: " 3 - 4096 0 0", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: " 4 - 4096 8 0", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: " 5 - 4096 64 0", 2024-09-21T09:51:12.946 INFO:tasks.workunit.client.0.smithi033.stdout: " 6 - 4096 128 0", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Critical Warning: 0x00", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Temperature: 25 Celsius", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare: 100%", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Percentage Used: 14%", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Read: 511,399,621 [261 TB]", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Written: 2,603,219,310 [1.33 PB]", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Read Commands: 7,827,858,467", 2024-09-21T09:51:12.947 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Write Commands: 43,981,992,044", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller Busy Time: 19", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "Power Cycles: 55,292", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "Power On Hours: 67,712", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "Unsafe Shutdowns: 780", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information Log Entries: 0", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "No Errors Logged", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "" 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "svn_revision": "5155", 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: "version": [ 2024-09-21T09:51:12.948 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": { 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: "current": 25 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: "user_capacity": { 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.949 INFO:tasks.workunit.client.0.smithi033.stdout: "069c6a6c-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "INTEL_SSDPEDMD400G4_069cdbaa-77ff-11ef-904e-ac1f6ba5d294": { 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "20240921-094503": { 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "device": { 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "info_name": "/dev/nvme0n1", 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "/dev/nvme0n1", 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "protocol": "NVMe", 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "nvme" 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "firmware_version": "8DV101H0", 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "host_id": "069c6a6c-77ff-11ef-904e-ac1f6ba5d294", 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "json_format_version": [ 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.950 INFO:tasks.workunit.client.0.smithi033.stdout: "local_time": { 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "asctime": "Sat Sep 21 09:45:03 2024 UTC", 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "time_t": 1726911903 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "logical_block_size": 512, 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "model_name": "INTEL SSDPEDMD400G4", 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_controller_id": 0, 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_ieee_oui_identifier": 6083300, 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_namespaces": [ 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "capacity": { 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.951 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "formatted_lba_size": 512, 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 1, 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "size": { 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "utilization": { 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_number_of_namespaces": 1, 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_pci_vendor": { 2024-09-21T09:51:12.952 INFO:tasks.workunit.client.0.smithi033.stdout: "id": 32902, 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "subsystem_id": 32902 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_add_log": { 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "Device stats": { 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_fw_downgrades": { 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "crc_error_count": { 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.953 INFO:tasks.workunit.client.0.smithi033.stdout: "end_to_end_error_detection_count": { 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "erase_fail_count": { 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "host_bytes_written": { 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 33753944 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "host_ctx_wear_used": { 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 1 2024-09-21T09:51:12.954 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "media_bytes_read": { 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "nand_bytes_written": { 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 56784585 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_stat_indicator": { 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "pll_lock_loss_count": { 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.955 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "program_fail_count": { 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "re_alloc_sectr_cnt": { 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "retry_buffer_overflow_count": { 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.956 INFO:tasks.workunit.client.0.smithi033.stdout: "soft_ecc_err_rate": { 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "thermal_throttle_status": { 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "cnt": 0, 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "pct": 0 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_host_reads": { 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.957 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_media_wear": { 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": "63.999023" 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "timed_workload_timer": { 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 100, 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 65535 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "unexp_power_loss": { 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 0, 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": 0 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "wear_leveling": { 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "normalized": 88, 2024-09-21T09:51:12.958 INFO:tasks.workunit.client.0.smithi033.stdout: "raw": { 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "avg": 3838, 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "max": 3855, 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 3821 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel Smart log": "nvme0n1" 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_smart_health_information_log": { 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare": 100, 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "available_spare_threshold": 10, 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "controller_busy_time": 18, 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "critical_warning": 0, 2024-09-21T09:51:12.959 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_read": 450935655, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "data_units_written": 2212098498, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "host_reads": 7207456630, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "host_writes": 38408967915, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "media_errors": 0, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "num_err_log_entries": 0, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "percentage_used": 12, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycles": 58006, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_hours": 58153, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": 23, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "unsafe_shutdowns": 1361 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_vendor": "intel", 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme_version": { 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "string": "<1.2", 2024-09-21T09:51:12.960 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "power_cycle_count": 58006, 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "power_on_time": { 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "hours": 58153 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "serial_number": "deleted", 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "smart_status": { 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "nvme": { 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "passed": true 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl": { 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "argv": [ 2024-09-21T09:51:12.961 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "-x", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "--json=o", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "/dev/nvme0n1" 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "build_info": "(local build)", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "exit_status": 0, 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "output": [ 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "smartctl 7.2 2020-12-30 r5155 [x86_64-linux-5.14.0-508.el9.x86_64] (local build)", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "Copyright (C) 2002-20, Bruce Allen, Christian Franke, www.smartmontools.org", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF INFORMATION SECTION ===", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "Model Number: INTEL SSDPEDMD400G4", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "Serial Number: deleted", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Version: 8DV101H0", 2024-09-21T09:51:12.962 INFO:tasks.workunit.client.0.smithi033.stdout: "PCI Vendor/Subsystem ID: 0x8086", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "IEEE OUI Identifier: 0x5cd2e4", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller ID: 0", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "NVMe Version: <1.2", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Number of Namespaces: 1", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Size/Capacity: 400,088,457,216 [400 GB]", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Namespace 1 Formatted LBA Size: 512", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Local Time is: Sat Sep 21 09:45:03 2024 UTC", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Firmware Updates (0x02): 1 Slot", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional Admin Commands (0x0006): Format Frmw_DL", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Optional NVM Commands (0x0006): Wr_Unc DS_Mngmt", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Log Page Attributes (0x02): Cmd_Eff_Lg", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Maximum Data Transfer Size: 32 Pages", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.963 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported Power States", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: "St Op Max Active Idle RL RT WL WT Ent_Lat Ex_Lat", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 25.00W - - 0 0 0 0 0 0", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: "Supported LBA Sizes (NSID 0x1)", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: "Id Fmt Data Metadt Rel_Perf", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: " 0 + 512 0 2", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: " 1 - 512 8 2", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: " 2 - 512 16 2", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: " 3 - 4096 0 0", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: " 4 - 4096 8 0", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: " 5 - 4096 64 0", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: " 6 - 4096 128 0", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: "=== START OF SMART DATA SECTION ===", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART overall-health self-assessment test result: PASSED", 2024-09-21T09:51:12.964 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "SMART/Health Information (NVMe Log 0x02)", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Critical Warning: 0x00", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Temperature: 23 Celsius", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare: 100%", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Available Spare Threshold: 10%", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Percentage Used: 12%", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Read: 450,935,655 [230 TB]", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Data Units Written: 2,212,098,498 [1.13 PB]", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Read Commands: 7,207,456,630", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Host Write Commands: 38,408,967,915", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Controller Busy Time: 18", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Power Cycles: 58,006", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Power On Hours: 58,153", 2024-09-21T09:51:12.965 INFO:tasks.workunit.client.0.smithi033.stdout: "Unsafe Shutdowns: 1,361", 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "Media and Data Integrity Errors: 0", 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information Log Entries: 0", 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "", 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "Error Information (NVMe Log 0x01, 16 of 64 entries)", 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "No Errors Logged", 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "" 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "platform_info": "x86_64-linux-5.14.0-508.el9.x86_64", 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "svn_revision": "5155", 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: "version": [ 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:12.966 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: "temperature": { 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: "current": 23 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: "user_capacity": { 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: "blocks": 781422768, 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 400088457216 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: "report": { 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: "balancer": { 2024-09-21T09:51:12.967 INFO:tasks.workunit.client.0.smithi033.stdout: "active": true, 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "mode": "" 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "channels": [ 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "basic", 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "crash", 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "device", 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "ident", 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "perf" 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "channels_available": [ 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "basic", 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "ident", 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "crash", 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "device", 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: "perf" 2024-09-21T09:51:12.968 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "collections_available": [ 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_base", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "device_base", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "crash_base", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "ident_base", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_perf", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_mds_metadata", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_usage", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_usage_by_class", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_rook_v01", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_memory_metrics", 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_options_bluestore" 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.969 INFO:tasks.workunit.client.0.smithi033.stdout: "collections_opted_in": [ 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_base", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "device_base", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "crash_base", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "ident_base", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_perf", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_mds_metadata", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_usage", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_usage_by_class", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_rook_v01", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_memory_metrics", 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "basic_pool_options_bluestore" 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "config": { 2024-09-21T09:51:12.970 INFO:tasks.workunit.client.0.smithi033.stdout: "active_changed": [ 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_debug", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_service_ticket_ttl", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "bdev_debug_aio", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "client_check_pool_perm", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "client_mount_gid", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "client_mount_uid", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "client_quota", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "container_image", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "daemonize", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mgr", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mon", 2024-09-21T09:51:12.971 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_ms", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_osd", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_paxos", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "keyring", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_block_size", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_cache_size", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_compression", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_log", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "leveldb_write_buffer_size", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_file", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_journald", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_stderr", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_allow_pool_delete", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_clock_drift_allowed", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_file", 2024-09-21T09:51:12.972 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_journald", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_stderr", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_data_avail_warn", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_election_default_strategy", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_host", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_max_pg_per_osd", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:51:12.973 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_bug", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_old_message", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "no_config_file", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_default_list", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_load_list", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_op_order", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_shutdown", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:51:12.974 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_profile", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_lim", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_res", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_best_effort_wgt", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_lim", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_res", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_background_recovery_wgt", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_lim", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_res", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_scheduler_client_wgt", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target_autotune", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue_cut_off", 2024-09-21T09:51:12.975 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recover_clone_overlap", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recovery_max_chunk", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_load_threshold", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_max_interval", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "public_addr", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "public_addrv", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "public_network", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd_default_features", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd_qos_exclude_ops", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "setgroup", 2024-09-21T09:51:12.976 INFO:tasks.workunit.client.0.smithi033.stdout: "setuser" 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster_changed": [ 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_allow_insecure_global_id_reclaim", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_debug", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_mon_ticket_ttl", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "auth_service_ticket_ttl", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "bdev_debug_aio", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "container_image", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_asserts_on_shutdown", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mgr", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_mon", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_ms", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_osd", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "debug_paxos", 2024-09-21T09:51:12.977 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_file", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_journald", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "log_to_stderr", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_join_fs", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/allow_ptrace", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/container_init", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/migration_current", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/cephadm/use_repo_digest", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/ALERTMANAGER_API_HOST", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/GRAFANA_API_SSL_VERIFY", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/GRAFANA_API_URL", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/PROMETHEUS_API_HOST", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/dashboard/ssl_server_port", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/orchestrator/orchestrator", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/channel_ident", 2024-09-21T09:51:12.978 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/channel_perf", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/description", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/enabled", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr/telemetry/organization", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_allow_pool_delete", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_clock_drift_allowed", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_cluster_log_to_file", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_data_avail_warn", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_election_default_strategy", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_max_pg_per_osd", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_mgr_mkfs_grace", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_prime_pg_temp", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_osd_reporter_subtree_level", 2024-09-21T09:51:12.979 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_pg_warn_max_object_skew", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_bytes_per_osd", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_reweight_min_pgs_per_osd", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_crush_straw_calc_version_zero", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_legacy_crush_tunables", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_osd_down_out_interval_zero", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_pool_pg_num_not_power_of_two", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "mon_warn_on_too_few_osds", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_bug", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "ms_die_on_old_message", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_default_list", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_class_load_list", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_misdirected_ops", 2024-09-21T09:51:12.980 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_op_order", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_pg_log_writeout", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_shutdown", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_cached_snaps", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_missing_on_start", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_debug_verify_stray_on_activate", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_deep_scrub_update_digest_min_age", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mclock_profile", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_memory_target_autotune", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_op_queue_cut_off", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_erasure_code_profile", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pool_default_pg_autoscale_mode", 2024-09-21T09:51:12.981 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recover_clone_overlap", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_recovery_max_chunk", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_load_threshold", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_scrub_max_interval", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_shutdown_pgref_assert", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "public_network", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_cache_enabled", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_enable_ops_log", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "rgw_enable_usage_log" 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "contact": null, 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "crashes": [], 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "created": "2024-09-21T09:41:48.442447Z", 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "crush": { 2024-09-21T09:51:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_algs": { 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "straw2": 3 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_sizes": { 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "2": 1, 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "4": 2 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket_types": { 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "1": 2, 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "11": 1 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "compat_weight_set": false, 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "device_classes": [ 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: 8 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.983 INFO:tasks.workunit.client.0.smithi033.stdout: "num_buckets": 6, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "num_devices": 8, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "num_rules": 1, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "num_types": 12, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "num_weight_sets": 0, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "tunables": { 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "allowed_bucket_algs": 54, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_local_fallback_tries": 0, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_local_tries": 0, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "choose_total_tries": 50, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_descend_once": 1, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_stable": 1, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "chooseleaf_vary_r": 1, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v2_rules": 0, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v3_rules": 0, 2024-09-21T09:51:12.984 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v4_buckets": 1, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "has_v5_rules": 0, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "legacy_tunables": 0, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "minimum_required_version": "jewel", 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "optimal_tunables": 1, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "profile": "jewel", 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables": 1, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables2": 1, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables3": 1, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "require_feature_tunables5": 1, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "straw_calc_version": 1 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "description": "upgrade test cluster", 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "fs": { 2024-09-21T09:51:12.985 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "feature_flags": { 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "enable_multiple": true, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "ever_enabled_multiple": true 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "filesystems": [ 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "approx_ctime": "2024-09", 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "balancer_enabled": false, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_caps": 0, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_dns": 10, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_inos": 13, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "cached_subtrees": 2, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "ever_allowed_features": 0, 2024-09-21T09:51:12.986 INFO:tasks.workunit.client.0.smithi033.stdout: "explicitly_allowed_features": 0, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "files": 0, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "max_mds": 1, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_data_pools": 1, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_in": 1, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_mds": 1, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": 0, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_standby_replay": 0, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_up": 1, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "snaps": 0, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "standby_count_wanted": 0 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_standby_mds": 1, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "total_num_mds": 2 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.987 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_stats": { 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 20652032, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 485016, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 532480, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 443, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 1316056, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 20, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 1125888, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 15103632, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 21184512 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.988 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 20135936, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 335664, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 761856, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 0, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 352, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 871200, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 12, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 861184, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 14684592, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 20135936 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.989 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 36085760, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 1286952, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 8192, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 10395648, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1131, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 6294264, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 21, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2397184, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 23477728, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 46481408 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.990 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 46407680, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 1289512, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 720896, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 4161536, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1739, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 10897336, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 23, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2979328, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 27899168, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 50569216 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.d48952f845b12c2ac6ef02c91277f5cb17a1e8a2": { 2024-09-21T09:51:12.991 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 35946496, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 941304, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2621440, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 901120, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 9469952, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 971, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 6417832, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 21, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 1620224, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 23444576, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 45416448 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:12.992 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.0": { 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 274145280, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 958600, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 630784, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1407, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 5708440, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 45, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2196224, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262529504, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 274776064 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.1": { 2024-09-21T09:51:12.993 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272809984, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 654816, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 16384, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1351, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 5594480, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2267392, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 261540784, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 272826368 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.994 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.2": { 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 271892480, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 774304, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 720896, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1266, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4727544, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2430976, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 261207144, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 272613376 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.3": { 2024-09-21T09:51:12.995 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 271466496, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 763360, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 376832, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1236, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4111872, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 45, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 1985792, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 261852960, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 271843328 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.4": { 2024-09-21T09:51:12.996 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272949248, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 701888, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 991232, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1246, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 5024032, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 47, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2094848, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262375968, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 273940480 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.5": { 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272621568, 2024-09-21T09:51:12.997 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 676104, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 24576, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 1105920, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1282, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4597432, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 47, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2168576, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262402368, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.6": { 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272957440, 2024-09-21T09:51:12.998 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 836952, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 24576, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 770048, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1274, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 4949232, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 46, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2210048, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262184120, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.7": { 2024-09-21T09:51:12.999 INFO:tasks.workunit.client.0.smithi033.stdout: "actual_memory_used": 272973824, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "central_cache_freelist": 867552, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "malloc_metadata": 2752512, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "page_heap_freelist": 0, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "released_to_os": 753664, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "spans_in_use": 1331, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "tcmalloc_page_size": 8192, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_cache_freelists": 5058032, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "thread_heaps_in_use": 45, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "transfer_cache_freelist": 2193408, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "use_by_application": 262102320, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "virtual_address_space_used": 273727488 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "hosts": { 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num": 2, 2024-09-21T09:51:13.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mds": 2, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mgr": 2, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_mon": 2, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_with_osd": 2 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "io_rate": { 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_stats_delta": { 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 0, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 0, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "stamp_delta": "12.002353", 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 254, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 338, 2024-09-21T09:51:13.003 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 56, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:13.004 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 0 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "leaderboard": false, 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "license": "sharing-1-0", 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "mempool": { 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.005 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.006 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.007 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.008 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.009 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 222471, 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 72 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.010 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 50488, 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 208 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.011 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8040, 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 137 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.012 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.013 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.014 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.015 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.016 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.017 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 149074, 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 33 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.018 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 856, 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 2 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.019 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.020 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:13.021 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.022 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.023 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.024 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.025 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 1090918, 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 347 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.026 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.027 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6688, 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 136 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12744, 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1546 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 5312, 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 103 2024-09-21T09:51:13.028 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.029 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.030 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.031 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.032 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.033 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 1124374, 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 333 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 352, 2024-09-21T09:51:13.034 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 4 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.035 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6688, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 136 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12744, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1546 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 10624, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 206 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.036 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.d48952f845b12c2ac6ef02c91277f5cb17a1e8a2": { 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.038 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.039 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.040 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.041 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 1082438, 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 349 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.042 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.043 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6688, 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 136 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12744, 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1546 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 5312, 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 103 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.044 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.0": { 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.045 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20272, 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 181 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.046 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 784, 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 7 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8688, 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 181 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20272, 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 181 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 5520, 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 69 2024-09-21T09:51:13.047 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 35195, 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 54289, 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9795 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 133960, 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 197 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 11064, 2024-09-21T09:51:13.048 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 707 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 31412, 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 197 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.049 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 386854, 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 84 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2442398, 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 196 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6072, 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 69 2024-09-21T09:51:13.050 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 521280, 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 45 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 76024, 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 151 2024-09-21T09:51:13.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235888, 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3530 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.052 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.1": { 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:13.053 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19152, 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 171 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.054 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 784, 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 7 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8208, 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 171 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19152, 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 171 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4720, 2024-09-21T09:51:13.055 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 59 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8192, 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 2 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 48202, 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8828 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 125120, 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 184 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.056 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 9060, 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 536 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 27144, 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 184 2024-09-21T09:51:13.057 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 736, 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 402121, 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 87 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12096, 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.058 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2419742, 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 189 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4752, 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 54 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 428608, 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 37 2024-09-21T09:51:13.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8528, 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 22 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235120, 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3494 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.060 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.061 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.2": { 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:13.062 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 18816, 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 168 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 672, 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 6 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8064, 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 168 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.063 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 18816, 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 168 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4400, 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 55 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4096, 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 45468, 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8444 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.064 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 119680, 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 176 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8328, 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 524 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 25600, 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 176 2024-09-21T09:51:13.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 736, 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 393929, 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 85 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12096, 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2422288, 2024-09-21T09:51:13.066 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 181 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4224, 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 48 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 359104, 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 31 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.067 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8840, 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 25 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235632, 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3518 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.068 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.3": { 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.069 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.070 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19264, 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 224, 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 2 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8256, 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19264, 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:13.071 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 3920, 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 49 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 46965, 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8725 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 123760, 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 182 2024-09-21T09:51:13.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 9112, 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 540 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 26464, 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 182 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.073 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2208, 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 266401, 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 54 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 30637, 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2453215, 2024-09-21T09:51:13.074 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 191 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4752, 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 54 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 428608, 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 37 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.075 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 10400, 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 20 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235888, 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3530 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.076 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.077 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.4": { 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.078 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19376, 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 173 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 672, 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 6 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.079 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8304, 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 173 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19376, 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 173 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 3440, 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 43 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4096, 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:13.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 48382, 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8923 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 126480, 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 186 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 9324, 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 542 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.081 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 27320, 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 186 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 736, 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 908797, 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 68 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.082 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 12096, 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2405354, 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 169 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4840, 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 55 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.083 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 451776, 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 39 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8944, 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 18 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235376, 2024-09-21T09:51:13.084 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3506 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.085 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.5": { 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.086 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19264, 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 336, 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.087 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8256, 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 19264, 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 172 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 3120, 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 39 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4096, 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 1 2024-09-21T09:51:13.088 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 49349, 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9012 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 127840, 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 188 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 9560, 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 544 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.089 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 28000, 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 188 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2208, 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 834798, 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 50 2024-09-21T09:51:13.090 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 30637, 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2411890, 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 183 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 5544, 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 63 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.091 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 474944, 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 41 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 13520, 2024-09-21T09:51:13.092 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 26 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235888, 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3530 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.093 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.6": { 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:13.094 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20048, 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 179 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 336, 2024-09-21T09:51:13.095 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8592, 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 179 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20048, 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 179 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2720, 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 34 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 18553, 2024-09-21T09:51:13.096 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 4 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 48626, 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9159 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 125120, 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 184 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 10544, 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 697 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.097 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 28340, 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 184 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 799341, 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 42 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.098 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2380108, 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 159 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 4928, 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 56 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.099 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 451776, 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 39 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 75192, 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 155 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235120, 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3494 2024-09-21T09:51:13.100 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.101 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.7": { 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "bloom_filter": { 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6304, 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 115 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_reader": { 2024-09-21T09:51:13.102 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 16896, 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 8 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs_file_writer": { 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 576, 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Blob": { 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20160, 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 180 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Buffer": { 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 1120, 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 10 2024-09-21T09:51:13.103 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_Extent": { 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 8640, 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 180 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_SharedBlob": { 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 20160, 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 180 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_alloc": { 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2240, 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 28 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.104 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_data": { 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 282624, 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 10 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_meta": { 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 52710, 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 9696 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_onode": { 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 132600, 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 195 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_cache_other": { 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 11348, 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 706 2024-09-21T09:51:13.105 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_fsck": { 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_inline_bl": { 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 30988, 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 195 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_txc": { 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing": { 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 183787, 2024-09-21T09:51:13.106 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 35 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore_writing_deferred": { 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_anon": { 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 2399253, 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 219 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "buffer_meta": { 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 6248, 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 71 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_co": { 2024-09-21T09:51:13.107 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 532864, 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 46 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_mapbl": { 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_pglog": { 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 78520, 2024-09-21T09:51:13.108 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 195 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap": { 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 235120, 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 3494 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap_mapping": { 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "pgmap": { 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_1": { 2024-09-21T09:51:13.109 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: "unittest_2": { 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes": 0, 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: "items": 0 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: "metadata": { 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 2 2024-09-21T09:51:13.110 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 1, 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 2 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 2 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.111 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 2 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 2 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 2 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 3 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.112 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 2, 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 1 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 3 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 3 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 3 2024-09-21T09:51:13.113 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 3 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 3 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "arch": { 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "x86_64": 8 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_version": { 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.114 INFO:tasks.workunit.client.0.smithi033.stdout: "cpu": { 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v3 @ 3.50GHz": 4, 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "Intel(R) Xeon(R) CPU E5-1620 v4 @ 3.50GHz": 4 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "distro": { 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "centos": 8 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "distro_description": { 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "CentOS Stream 9": 8 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_description": { 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024": 8 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "kernel_version": { 2024-09-21T09:51:13.115 INFO:tasks.workunit.client.0.smithi033.stdout: "5.14.0-508.el9.x86_64": 8 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: "os": { 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: "Linux": 8 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_objectstore": { 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": 8 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: "rotational": { 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: "0": 8 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:13.116 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "features": { 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "optional": [], 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "persistent": [ 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "kraken", 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "luminous", 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "mimic", 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "osdmap-prune", 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "nautilus", 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "octopus", 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "pacific", 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "elector-pinging", 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "quincy" 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.117 INFO:tasks.workunit.client.0.smithi033.stdout: "ipv4_addr_mons": 6, 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "ipv6_addr_mons": 0, 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "min_mon_release": 17, 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "v1_addr_mons": 3, 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "v2_addr_mons": 3 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "organization": "ceph-qa", 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster_network": false, 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 8, 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "require_min_compat_client": "luminous", 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "require_osd_release": "quincy" 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_perf_histograms": [ 2024-09-21T09:51:13.118 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "axes": [ 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Latency (usec)", 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 100000, 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 99999]", 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "[100000, 199999]", 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "[200000, 399999]", 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "[400000, 799999]", 2024-09-21T09:51:13.119 INFO:tasks.workunit.client.0.smithi033.stdout: "[800000, 1599999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[1600000, 3199999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[3200000, 6399999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[6400000, 12799999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[12800000, 25599999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[25600000, 51199999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[51200000, 102399999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[102400000, 204799999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[204800000, 409599999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[409600000, 819199999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[819200000, 1638399999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[1638400000, 3276799999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[3276800000, 6553599999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[6553600000, 13107199999]", 2024-09-21T09:51:13.120 INFO:tasks.workunit.client.0.smithi033.stdout: "[13107200000, 26214399999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[26214400000, 52428799999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[52428800000, 104857599999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[104857600000, 209715199999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[209715200000, 419430399999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[419430400000, 838860799999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "[53687091200000, None]" 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.121 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Request size (bytes)", 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 512, 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 511]", 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "[512, 1023]", 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "[1024, 2047]", 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "[2048, 4095]", 2024-09-21T09:51:13.122 INFO:tasks.workunit.client.0.smithi033.stdout: "[4096, 8191]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[8192, 16383]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[16384, 32767]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[32768, 65535]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[65536, 131071]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[131072, 262143]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[262144, 524287]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[524288, 1048575]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[1048576, 2097151]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[2097152, 4194303]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[4194304, 8388607]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[8388608, 16777215]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[16777216, 33554431]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[33554432, 67108863]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[67108864, 134217727]", 2024-09-21T09:51:13.123 INFO:tasks.workunit.client.0.smithi033.stdout: "[134217728, 268435455]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[268435456, 536870911]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[536870912, 1073741823]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[1073741824, 2147483647]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[2147483648, 4294967295]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[4294967296, 8589934591]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[8589934592, 17179869183]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[17179869184, 34359738367]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[34359738368, 68719476735]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[68719476736, 137438953471]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[137438953472, 274877906943]", 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "[274877906944, None]" 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.124 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "osds": [ 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 0, 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.125 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.126 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 1, 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.127 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.128 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.129 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 2, 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.130 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.131 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 3, 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.132 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.133 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.134 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 4, 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.135 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.136 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 5, 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.137 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.138 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.139 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 6, 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.140 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.141 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 7, 2024-09-21T09:51:13.142 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.143 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.144 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: "axes": [ 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Latency (usec)", 2024-09-21T09:51:13.145 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 100000, 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 99999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[100000, 199999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[200000, 399999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[400000, 799999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[800000, 1599999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[1600000, 3199999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[3200000, 6399999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[6400000, 12799999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[12800000, 25599999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[25600000, 51199999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[51200000, 102399999]", 2024-09-21T09:51:13.146 INFO:tasks.workunit.client.0.smithi033.stdout: "[102400000, 204799999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[204800000, 409599999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[409600000, 819199999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[819200000, 1638399999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[1638400000, 3276799999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[3276800000, 6553599999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[6553600000, 13107199999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[13107200000, 26214399999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[26214400000, 52428799999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[52428800000, 104857599999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[104857600000, 209715199999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[209715200000, 419430399999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[419430400000, 838860799999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:51:13.147 INFO:tasks.workunit.client.0.smithi033.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "[53687091200000, None]" 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Request size (bytes)", 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 512, 2024-09-21T09:51:13.148 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 511]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[512, 1023]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[1024, 2047]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[2048, 4095]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[4096, 8191]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[8192, 16383]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[16384, 32767]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[32768, 65535]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[65536, 131071]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[131072, 262143]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[262144, 524287]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[524288, 1048575]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[1048576, 2097151]", 2024-09-21T09:51:13.149 INFO:tasks.workunit.client.0.smithi033.stdout: "[2097152, 4194303]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[4194304, 8388607]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[8388608, 16777215]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[16777216, 33554431]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[33554432, 67108863]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[67108864, 134217727]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[134217728, 268435455]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[268435456, 536870911]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[536870912, 1073741823]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[1073741824, 2147483647]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[2147483648, 4294967295]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[4294967296, 8589934591]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[8589934592, 17179869183]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[17179869184, 34359738367]", 2024-09-21T09:51:13.150 INFO:tasks.workunit.client.0.smithi033.stdout: "[34359738368, 68719476735]", 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "[68719476736, 137438953471]", 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "[137438953472, 274877906943]", 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "[274877906944, None]" 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "osds": [ 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 0, 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.151 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.152 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.153 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 1, 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.154 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.155 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.156 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 2, 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.157 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.158 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 3, 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.159 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.160 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.161 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 4, 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.162 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.163 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 5, 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.164 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.165 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.166 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 6, 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.167 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.168 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.169 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 7, 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.170 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.171 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "axes": [ 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:13.172 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Latency (usec)", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 100000, 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 99999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[100000, 199999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[200000, 399999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[400000, 799999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[800000, 1599999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[1600000, 3199999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[3200000, 6399999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[6400000, 12799999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[12800000, 25599999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[25600000, 51199999]", 2024-09-21T09:51:13.173 INFO:tasks.workunit.client.0.smithi033.stdout: "[51200000, 102399999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[102400000, 204799999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[204800000, 409599999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[409600000, 819199999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[819200000, 1638399999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[1638400000, 3276799999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[3276800000, 6553599999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[6553600000, 13107199999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[13107200000, 26214399999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[26214400000, 52428799999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[52428800000, 104857599999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[104857600000, 209715199999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[209715200000, 419430399999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[419430400000, 838860799999]", 2024-09-21T09:51:13.174 INFO:tasks.workunit.client.0.smithi033.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "[53687091200000, None]" 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:13.175 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Request size (bytes)", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 512, 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 511]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[512, 1023]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[1024, 2047]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[2048, 4095]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[4096, 8191]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[8192, 16383]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[16384, 32767]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[32768, 65535]", 2024-09-21T09:51:13.176 INFO:tasks.workunit.client.0.smithi033.stdout: "[65536, 131071]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[131072, 262143]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[262144, 524287]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[524288, 1048575]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[1048576, 2097151]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[2097152, 4194303]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[4194304, 8388607]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[8388608, 16777215]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[16777216, 33554431]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[33554432, 67108863]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[67108864, 134217727]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[134217728, 268435455]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[268435456, 536870911]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[536870912, 1073741823]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[1073741824, 2147483647]", 2024-09-21T09:51:13.177 INFO:tasks.workunit.client.0.smithi033.stdout: "[2147483648, 4294967295]", 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "[4294967296, 8589934591]", 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "[8589934592, 17179869183]", 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "[17179869184, 34359738367]", 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "[34359738368, 68719476735]", 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "[68719476736, 137438953471]", 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "[137438953472, 274877906943]", 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "[274877906944, None]" 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "osds": [ 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.178 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 0, 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.179 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.180 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 1, 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.181 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.182 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.183 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 2, 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.184 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.185 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 3, 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.186 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.187 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 4, 2024-09-21T09:51:13.188 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.189 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.190 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 5, 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.191 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.192 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 6, 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.193 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.194 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.195 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 7, 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.196 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.197 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "axes": [ 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:13.198 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Latency (usec)", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 100000, 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 99999]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[100000, 199999]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[200000, 399999]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[400000, 799999]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[800000, 1599999]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[1600000, 3199999]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[3200000, 6399999]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[6400000, 12799999]", 2024-09-21T09:51:13.199 INFO:tasks.workunit.client.0.smithi033.stdout: "[12800000, 25599999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[25600000, 51199999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[51200000, 102399999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[102400000, 204799999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[204800000, 409599999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[409600000, 819199999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[819200000, 1638399999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[1638400000, 3276799999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[3276800000, 6553599999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[6553600000, 13107199999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[13107200000, 26214399999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[26214400000, 52428799999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[52428800000, 104857599999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[104857600000, 209715199999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[209715200000, 419430399999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[419430400000, 838860799999]", 2024-09-21T09:51:13.200 INFO:tasks.workunit.client.0.smithi033.stdout: "[838860800000, 1677721599999]", 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "[1677721600000, 3355443199999]", 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "[3355443200000, 6710886399999]", 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "[6710886400000, 13421772799999]", 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "[13421772800000, 26843545599999]", 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "[26843545600000, 53687091199999]", 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "[53687091200000, None]" 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "buckets": 32, 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "min": 0, 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "name": "Request size (bytes)", 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "quant_size": 512, 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "ranges": [ 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "[None, -1]", 2024-09-21T09:51:13.201 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 511]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[512, 1023]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[1024, 2047]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[2048, 4095]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[4096, 8191]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[8192, 16383]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[16384, 32767]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[32768, 65535]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[65536, 131071]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[131072, 262143]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[262144, 524287]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[524288, 1048575]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[1048576, 2097151]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[2097152, 4194303]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[4194304, 8388607]", 2024-09-21T09:51:13.202 INFO:tasks.workunit.client.0.smithi033.stdout: "[8388608, 16777215]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[16777216, 33554431]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[33554432, 67108863]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[67108864, 134217727]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[134217728, 268435455]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[268435456, 536870911]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[536870912, 1073741823]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[1073741824, 2147483647]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[2147483648, 4294967295]", 2024-09-21T09:51:13.203 INFO:tasks.workunit.client.0.smithi033.stdout: "[4294967296, 8589934591]", 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: "[8589934592, 17179869183]", 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: "[17179869184, 34359738367]", 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: "[34359738368, 68719476735]", 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: "[68719476736, 137438953471]", 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: "[137438953472, 274877906943]", 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: "[274877906944, None]" 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: "scale_type": "log2" 2024-09-21T09:51:13.205 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "osds": [ 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 0, 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.206 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.207 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 1, 2024-09-21T09:51:13.208 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 8, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 7, 3, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 12, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.209 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.210 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 2, 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.211 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.212 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 3, 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.213 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.214 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.215 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 4, 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.216 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.217 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.218 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 5, 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.219 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.220 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.221 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 6, 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.222 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.223 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "osd_id": 7, 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "values": [ 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.224 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.225 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]", 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]" 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: ] 2024-09-21T09:51:13.226 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "perf_counters": { 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "mds.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "mds": { 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "caps": { 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_flush_ack": { 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_flushsnap_ack": { 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.227 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_grant": { 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_revoke": { 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "ceph_cap_op_trunc": { 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "dir_commit": { 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "dir_fetch": { 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "dir_merge": { 2024-09-21T09:51:13.228 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "dir_split": { 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "exported_inodes": { 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "forward": { 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_cap_release": { 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_caps": { 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.229 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_caps_dirty": { 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_inode_file_caps": { 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "imported_inodes": { 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "inodes": { 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.230 INFO:tasks.workunit.client.0.smithi033.stdout: "inodes_expired": { 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "inodes_pinned": { 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "inodes_with_caps": { 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "load_cent": { 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "openino_dir_fetch": { 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.231 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "process_request_cap_release": { 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "reply_latency": { 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 147843 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "request": { 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "root_rbytes": { 2024-09-21T09:51:13.232 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: "root_rfiles": { 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: "root_rsnaps": { 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_reply": { 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.233 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: "subtrees": { 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_cache": { 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_enqueue_scrub": { 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_exportdir": { 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.234 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_flush": { 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_fragmentdir": { 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_fragstats": { 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.235 INFO:tasks.workunit.client.0.smithi033.stdout: "ireq_inodestats": { 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_recovering_enqueued": { 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_recovering_prioritized": { 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_recovering_processing": { 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.236 INFO:tasks.workunit.client.0.smithi033.stdout: "num_strays": { 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_strays_delayed": { 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "num_strays_enqueuing": { 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_completed": { 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_started": { 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.237 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "strays_created": { 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "strays_enqueued": { 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "strays_migrated": { 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "strays_reintegrated": { 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_log": { 2024-09-21T09:51:13.238 INFO:tasks.workunit.client.0.smithi033.stdout: "ev": { 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: "evadd": { 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: "evex": { 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: "evexd": { 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.239 INFO:tasks.workunit.client.0.smithi033.stdout: "evexg": { 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "evtrm": { 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "jlat": { 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 9318416 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "replayed": { 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.240 INFO:tasks.workunit.client.0.smithi033.stdout: "seg": { 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "segadd": { 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "segex": { 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "segexd": { 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "segexg": { 2024-09-21T09:51:13.241 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "segtrm": { 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_mem": { 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "cap": { 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "cap+": { 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "cap-": { 2024-09-21T09:51:13.242 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "dir": { 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "dir+": { 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "dir-": { 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "dn": { 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "dn+": { 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:13.243 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "dn-": { 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "heap": { 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 207068 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "ino": { 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "ino+": { 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "ino-": { 2024-09-21T09:51:13.244 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "rss": { 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 38332 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_server": { 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "cap_acquisition_throttle": { 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "cap_revoke_eviction": { 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_request": { 2024-09-21T09:51:13.245 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_client_session": { 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 16 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "handle_peer_request": { 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "req_create_latency": { 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "req_getattr_latency": { 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.246 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 147843 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: "req_getfilelock_latency": { 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: "req_getvxattr_latency": { 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: "req_link_latency": { 2024-09-21T09:51:13.247 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookup_latency": { 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookuphash_latency": { 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.248 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookupino_latency": { 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookupname_latency": { 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookupparent_latency": { 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lookupsnap_latency": { 2024-09-21T09:51:13.249 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "req_lssnap_latency": { 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "req_mkdir_latency": { 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "req_mknod_latency": { 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.250 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "req_mksnap_latency": { 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "req_open_latency": { 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "req_readdir_latency": { 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.251 INFO:tasks.workunit.client.0.smithi033.stdout: "req_rename_latency": { 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "req_renamesnap_latency": { 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "req_rmdir_latency": { 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.252 INFO:tasks.workunit.client.0.smithi033.stdout: "req_rmsnap_latency": { 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "req_rmxattr_latency": { 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setattr_latency": { 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.253 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setdirlayout_latency": { 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setfilelock_latency": { 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setlayout_latency": { 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.254 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "req_setxattr_latency": { 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "req_symlink_latency": { 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "req_unlink_latency": { 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.255 INFO:tasks.workunit.client.0.smithi033.stdout: "mds_sessions": { 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "average_load": { 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "avg_session_uptime": { 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 87 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "mdthresh_evicted": { 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "session_add": { 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "session_count": { 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.256 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "session_remove": { 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "sessions_open": { 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "sessions_stale": { 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "total_load": { 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.257 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.258 INFO:tasks.workunit.client.0.smithi033.stdout: "purge_queue": { 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executed": { 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executing": { 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executing_high_water": { 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.259 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executing_ops": { 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_executing_ops_high_water": { 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: "pq_item_in_journal": { 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.260 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.734a9911f33289fd62fac5f7ea97492252ac4ce0": { 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: "election_call": { 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: "election_lose": { 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: "election_win": { 2024-09-21T09:51:13.261 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: "num_elections": { 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": { 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 11 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: "session_add": { 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: "session_rm": { 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 52 2024-09-21T09:51:13.262 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "session_trim": { 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "paxos": { 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "accept_timeout": { 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "begin": { 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 846 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_bytes": { 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 846, 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4065103 2024-09-21T09:51:13.263 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_keys": { 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_latency": { 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 846, 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 121997422 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "collect": { 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.264 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_bytes": { 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 146 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_keys": { 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_latency": { 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.265 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_timeout": { 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_uncommitted": { 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: "commit": { 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 847 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_bytes": { 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.266 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_keys": { 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_latency": { 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_ack_timeout": { 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.267 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_timeout": { 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn": { 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn_latency": { 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh": { 2024-09-21T09:51:13.268 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 847 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh_latency": { 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 847, 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1850960838 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: "restart": { 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state": { 2024-09-21T09:51:13.269 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_bytes": { 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_keys": { 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: "start_leader": { 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.270 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "start_peon": { 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state": { 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 847 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_bytes": { 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 847, 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7971301 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_keys": { 2024-09-21T09:51:13.271 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 847, 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5693 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_latency": { 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 847, 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 283993992 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache": { 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1020054731 2024-09-21T09:51:13.272 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 43859968 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33595392 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2147483648 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10264576 2024-09-21T09:51:13.273 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:full": { 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.274 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277021456 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 271262 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.275 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.276 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.277 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 70834514 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:inc": { 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.278 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277021456 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 41216 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.279 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.280 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71064560 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:kv": { 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 322961408 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.281 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 184680971 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.282 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.283 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.284 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71171573 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb": { 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "compact": { 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_completed": { 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lasted": { 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 28310214 2024-09-21T09:51:13.285 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_len": { 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_merge": { 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_running": { 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "get_latency": { 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 36184, 2024-09-21T09:51:13.286 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 82964913 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.287 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_latency": { 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_sync_latency": { 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1843, 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 504995419 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.81c1e4013e7fb0a86cba873269ee01d73616d8ce": { 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:13.288 INFO:tasks.workunit.client.0.smithi033.stdout: "election_call": { 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "election_lose": { 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "election_win": { 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_elections": { 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 9 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": { 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "session_add": { 2024-09-21T09:51:13.289 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "session_rm": { 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 186 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "session_trim": { 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "paxos": { 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "accept_timeout": { 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "begin": { 2024-09-21T09:51:13.290 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 972 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_bytes": { 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5114206 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_keys": { 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2916 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_latency": { 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 173997229 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "collect": { 2024-09-21T09:51:13.291 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_bytes": { 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_keys": { 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_latency": { 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.292 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_timeout": { 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_uncommitted": { 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "commit": { 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 972 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_bytes": { 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5044280 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.293 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_keys": { 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5530 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_latency": { 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 972, 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 402255956 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_ack_timeout": { 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_timeout": { 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.294 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn": { 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn_latency": { 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5999904 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh": { 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 974 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh_latency": { 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 974, 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2592958617 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "restart": { 2024-09-21T09:51:13.295 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state": { 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_bytes": { 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1050 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_keys": { 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "start_leader": { 2024-09-21T09:51:13.296 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "start_peon": { 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state": { 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_bytes": { 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_keys": { 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.297 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_latency": { 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache": { 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1020054731 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 47947776 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42426368 2024-09-21T09:51:13.298 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2147483648 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5521408 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:full": { 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.299 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277022519 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270832 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.300 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.301 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 70833881 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:inc": { 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.302 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277022519 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 38813 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.303 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.304 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71065900 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:kv": { 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 322961408 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.305 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 184681679 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.306 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.307 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71170865 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb": { 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "compact": { 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_completed": { 2024-09-21T09:51:13.308 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lasted": { 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33964203 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_len": { 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_merge": { 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_running": { 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.309 INFO:tasks.workunit.client.0.smithi033.stdout: "get_latency": { 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 43614, 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 171857452 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.310 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_latency": { 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_sync_latency": { 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2114, 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 669135547 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.311 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "mon.d48952f845b12c2ac6ef02c91277f5cb17a1e8a2": { 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "election_call": { 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "election_lose": { 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "election_win": { 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "num_elections": { 2024-09-21T09:51:13.312 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_sessions": { 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "session_add": { 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "session_rm": { 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "session_trim": { 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "paxos": { 2024-09-21T09:51:13.313 INFO:tasks.workunit.client.0.smithi033.stdout: "accept_timeout": { 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "begin": { 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 836 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_bytes": { 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 836, 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4038275 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_keys": { 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "begin_latency": { 2024-09-21T09:51:13.314 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 836, 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 240996161 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "collect": { 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_bytes": { 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_keys": { 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_latency": { 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.315 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2999952 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_timeout": { 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "collect_uncommitted": { 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "commit": { 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 837 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_bytes": { 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.316 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_keys": { 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "commit_latency": { 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_ack_timeout": { 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "lease_timeout": { 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.317 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn": { 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "new_pn_latency": { 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh": { 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 837 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "refresh_latency": { 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 837, 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1981968369 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "restart": { 2024-09-21T09:51:13.318 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state": { 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_bytes": { 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "share_state_keys": { 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "start_leader": { 2024-09-21T09:51:13.319 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "start_peon": { 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state": { 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 837 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_bytes": { 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 837, 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7919743 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_keys": { 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 837, 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5618 2024-09-21T09:51:13.320 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "store_state_latency": { 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 837, 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 517991733 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache": { 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1020054731 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42795008 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.321 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 32006144 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2147483648 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10788864 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:full": { 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.322 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277021456 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 271262 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.323 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.324 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 70834514 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.325 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:inc": { 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 348127232 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 277021456 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.326 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 41216 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.327 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71064560 2024-09-21T09:51:13.328 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "prioritycache:kv": { 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 322961408 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.329 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 184680971 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.330 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.331 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71171573 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb": { 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "compact": { 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_completed": { 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lasted": { 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 32763828 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.332 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_len": { 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_queue_merge": { 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_running": { 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "get_latency": { 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 35626, 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 92285215 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_delay_time": { 2024-09-21T09:51:13.333 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_memtable_time": { 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_pre_and_post_time": { 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_write_wal_time": { 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.334 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_latency": { 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "submit_sync_latency": { 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1794, 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 807289558 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.0": { 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:13.335 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.336 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2375680 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.337 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1581056 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1441792 2024-09-21T09:51:13.338 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:13.339 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:13.340 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.341 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.342 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.343 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3891200 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:13.344 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1294336 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 186166 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:13.345 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 25, 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 55958 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 335, 2024-09-21T09:51:13.346 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71401603 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 323, 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 9584339 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 335, 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 406733 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 335, 2024-09-21T09:51:13.347 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71808336 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33694 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8910 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.348 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10649 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 725 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.349 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 226 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 110, 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3514004 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 264, 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 274372 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 154, 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3285427 2024-09-21T09:51:13.350 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 39402 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5477539 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 99, 2024-09-21T09:51:13.351 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6472508 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 99, 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 527220255598 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 99, 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 133043675413 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 51005392880 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.352 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 48704 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 399797 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 151770600 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.353 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 151121 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 69608566 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 25687679 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 900429 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:13.354 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 59, 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 285208 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 253003984 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 29516713 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 440, 2024-09-21T09:51:13.355 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2845196 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1862399738 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 272023552 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 271237120 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.356 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3138477772 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 786432 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.357 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85552758 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.358 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 28672 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.359 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73802122 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.360 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641645685 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.361 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.362 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71383867 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.363 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 57035172 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.364 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68793484 2024-09-21T09:51:13.365 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641645685 2024-09-21T09:51:13.366 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.367 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 278698 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71107297 2024-09-21T09:51:13.368 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.369 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:13.370 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 604 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45997103 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 7, 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3391356 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.371 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15749679 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:13.372 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.373 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.374 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 604 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:13.375 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45997103 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 7, 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3391356 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15749679 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.376 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305782784 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.377 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.1": { 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.378 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2031616 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.379 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1384448 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.380 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1245184 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.381 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.382 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.383 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.384 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:13.385 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3350528 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.386 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 700416 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 210064 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.387 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 25, 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 58135 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:13.388 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 287, 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 59665849 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 282, 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7098878 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 287, 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 365579 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 287, 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 60031428 2024-09-21T09:51:13.389 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 62719 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15591 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.390 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13797 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 669 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:13.391 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 232 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 115, 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2623473 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 269, 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 228044 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 154, 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2809408 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:13.392 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 102776 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4983810 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 94, 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7018852 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:13.393 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 93, 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 375555613811 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 94, 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 109497588840 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 353, 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 54681211745 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 353, 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37801 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.394 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 379437 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 96649729 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 114438 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 45843521 2024-09-21T09:51:13.395 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 18018332 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 309839 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 53, 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 254818 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 165923839 2024-09-21T09:51:13.396 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 22284785 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 354, 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3388579 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1862399738 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270073856 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.397 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269901824 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3138477772 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 172032 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:13.398 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85555470 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.399 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8192 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.400 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73819890 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:13.401 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641666032 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.402 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.403 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71363520 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.404 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 57036980 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.405 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.406 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68791676 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.407 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641666032 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.408 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 254030 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.409 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71111618 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.410 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 97 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:13.411 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1640827 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 97, 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 637800589 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 602 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.412 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 97, 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 36592411 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 97, 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 132882259 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 18 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 18, 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3160177 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.413 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 602 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 18, 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2769359 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 18, 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2293151 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.414 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 12, 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13771748 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 12, 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5330101 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 12, 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13128135 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.416 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1640827 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 67, 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 620868664 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 67, 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 28492951 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:13.417 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 67, 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 117460973 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 590368 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.418 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305192960 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.2": { 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:13.419 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.420 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1753088 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:13.421 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1187840 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1048576 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.422 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:13.423 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:13.424 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:13.425 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.426 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2875392 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.427 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 688128 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 175858 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.428 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 9, 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 11904 2024-09-21T09:51:13.429 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 239, 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 60236767 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 228, 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12943737 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 239, 2024-09-21T09:51:13.430 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 418992 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 239, 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 60655759 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37463 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:13.431 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1597 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4036 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 517 2024-09-21T09:51:13.432 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 205 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 100, 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1225862 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 236, 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 274489 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 136, 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1044162 2024-09-21T09:51:13.433 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 82831 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6118277 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:13.434 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10109099 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 89, 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 373715777747 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 153321542323 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 288, 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 63394314049 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:13.435 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 288, 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 29551 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 433293 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 135486112 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 96564 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.436 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33801186 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 16648043 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 309660 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 50, 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 287276 2024-09-21T09:51:13.437 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192527012 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 20043992 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 289, 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1979043 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.438 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1862399738 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269860864 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269049856 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3138477772 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.439 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 811008 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85556399 2024-09-21T09:51:13.440 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.442 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73823057 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641672995 2024-09-21T09:51:13.443 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.444 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71356557 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.445 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 57037599 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.446 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.447 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68791057 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.448 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641672995 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.450 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 242654 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71116031 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.451 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.452 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 31 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:13.453 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 52019984 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4319689 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:13.454 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37957056 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:13.455 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.456 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:13.457 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:13.458 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 52019984 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4319689 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 10, 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37957056 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.459 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305180672 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.460 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.3": { 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.461 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1101824 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.462 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 827392 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 688128 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.463 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:13.464 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:13.465 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.467 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:13.468 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1863680 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 704512 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:13.469 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.470 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5348 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 151, 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 41844588 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 148, 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8633007 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 151, 2024-09-21T09:51:13.471 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 239278 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 151, 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42083866 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 40594 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.472 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 864 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8854 2024-09-21T09:51:13.473 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 335 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 213 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 80, 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 716368 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 139, 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 216867 2024-09-21T09:51:13.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 59, 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 399739 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5555149 2024-09-21T09:51:13.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 51, 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3883301 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 48, 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 373991339684 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 51, 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71425295254 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 235, 2024-09-21T09:51:13.476 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 558158 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 235, 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 25768 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 251769 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 134071728 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:13.477 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 66707 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 77496601 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12983832 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312073 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.478 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 42, 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 201223 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 230387327 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15304958 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:13.479 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 238, 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3576473 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1862399738 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269090816 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 268550144 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3138477772 2024-09-21T09:51:13.480 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 540672 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.481 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85556173 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.482 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.483 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73827379 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.484 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641671299 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.485 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.486 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71358253 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.487 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 57037448 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.488 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.489 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68791208 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.490 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641671299 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.491 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6959 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 246349 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.492 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71107073 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.493 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.494 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.495 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 735637 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:13.496 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:13.498 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.499 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.500 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 735637 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:13.501 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305192960 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.4": { 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:13.502 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.503 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1327104 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:13.504 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1019904 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 880640 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.505 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:13.506 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.507 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.508 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.509 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.510 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2281472 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 708608 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 8, 2024-09-21T09:51:13.511 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 321307 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:13.512 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 7, 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 22264 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 198, 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42148142 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.513 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 192, 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7452239 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 198, 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 215856 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 198, 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42363998 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 34342 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4747 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.515 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12969 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 376 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 232 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 94, 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3100301 2024-09-21T09:51:13.516 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 158, 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 228182 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 64, 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2413987 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 106043 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.517 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5681038 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 41, 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2890771 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 40, 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 181315760770 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:13.518 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 41, 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 64593627717 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 270, 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 521458 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 270, 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 31097 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 335359 2024-09-21T09:51:13.519 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 83015799 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85845 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 40190223 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:13.520 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12794179 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 310375 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 39, 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 214523 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 142055651 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:13.521 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15356206 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 271, 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1991897 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1861455963 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.522 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270139392 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270032896 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3137367449 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 106496 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:13.523 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85498945 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.524 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.525 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73880511 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:13.526 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641242091 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.527 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.528 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71787461 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:13.529 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 56999297 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.530 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.531 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68829359 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.532 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641242091 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.533 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 253659 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.534 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71535930 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.535 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 39 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.536 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 22 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5882849 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.537 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 728215 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5772701 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.538 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.539 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.540 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1 2024-09-21T09:51:13.541 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 22 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5882849 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.542 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 728215 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5772701 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.543 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305201152 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.5": { 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:13.544 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.545 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1232896 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.546 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 929792 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 790528 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.547 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:13.548 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.549 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:13.550 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.551 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:13.552 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2097152 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 704512 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 6, 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 208562 2024-09-21T09:51:13.553 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:13.554 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5311 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 176, 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42276933 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.555 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 168, 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 9086784 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 176, 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 170584 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 176, 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42447517 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:13.556 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 31748 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 958 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.557 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7256 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 345 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:13.558 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 235 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 85, 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 801632 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 139, 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 179244 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 54, 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 547107 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.559 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 4, 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 75787 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3297153 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.560 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 42, 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3292403 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 39, 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 309073604821 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 42, 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 66506969968 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 272, 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3561876644 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.561 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 272, 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 35436 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 227203 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 103271374 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 84263 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.562 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71342577 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13460429 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312078 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 34, 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 161256 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.563 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 191637146 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15854454 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 275, 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1867140 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.564 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1861455963 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270974976 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269991936 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3137367449 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 983040 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:13.565 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85498787 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.566 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.567 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73880669 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:13.568 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641240905 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.569 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.570 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71788647 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.571 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 56999191 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.572 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.573 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68829465 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.574 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641240905 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.575 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6964 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 254904 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.576 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71528907 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.577 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.578 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 41 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.579 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:13.580 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.581 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.582 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.583 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.584 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.585 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.586 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 590368 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.587 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305192960 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.6": { 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.588 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.589 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1277952 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.590 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 974848 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 835584 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.591 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:13.592 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:13.593 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:13.594 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.595 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:13.596 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2187264 2024-09-21T09:51:13.597 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1286144 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.598 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:13.599 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 11, 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 54192 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42087380 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 173, 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 16315080 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.600 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 157888 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 42245268 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 27344 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.601 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4569 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.602 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 8100 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 374 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 240 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 86, 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3204171 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 134, 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 207656 2024-09-21T09:51:13.603 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 48, 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2946894 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.604 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2795077 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 36, 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2511161 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 36, 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 386962683653 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 36, 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67818634494 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:13.605 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 459121 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 31185 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 204826 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 207446838 2024-09-21T09:51:13.606 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 82901 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 89091144 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12025409 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.607 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 902093 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 29, 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 125974 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 311600797 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 19156167 2024-09-21T09:51:13.608 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 278, 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1921753 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1861455963 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270974976 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270082048 2024-09-21T09:51:13.609 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3137367449 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 892928 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.610 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85498308 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.611 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.612 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73872956 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.613 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641237316 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.614 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.615 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.616 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71792236 2024-09-21T09:51:13.617 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.617 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.617 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:13.617 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.617 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 56998872 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.618 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.619 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.620 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68829784 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.621 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641237316 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.622 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7110 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 7110 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.623 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 247433 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.624 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71532711 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.625 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 39 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:13.626 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.627 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2804 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 13, 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 64976516 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.628 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 15, 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6275495 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 13, 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 30730878 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.629 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.630 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.631 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.632 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2804 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 13, 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 64976516 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 15, 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6275495 2024-09-21T09:51:13.633 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 13, 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 30730878 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 590368 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.634 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305774592 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "osd.7": { 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "bluefs": { 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_fallback": { 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_slow_size_fallback": { 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.635 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_db": { 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 65536 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_main": { 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit_wal": { 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_slow": { 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_sst": { 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12288 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.636 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_written_wal": { 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1257472 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lat": { 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "compact_lock_lat": { 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "db_total_bytes": { 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "db_used_bytes": { 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 304480256 2024-09-21T09:51:13.637 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "log_bytes": { 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 974848 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "logged_bytes": { 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 835584 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_db": { 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 312868864 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_slow": { 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.638 INFO:tasks.workunit.client.0.smithi033.stdout: "max_bytes_wal": { 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "num_files": { 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "read_bytes": { 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 195697 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "read_count": { 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 23 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes": { 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1269760 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_db": { 2024-09-21T09:51:13.639 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1273856 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_slow": { 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_bytes_wal": { 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "read_disk_count": { 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_bytes": { 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4789 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "read_prefetch_count": { 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4 2024-09-21T09:51:13.640 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_bytes": { 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_buffer_count": { 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_bytes": { 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5589 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_count": { 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 24 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes": { 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.641 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_db": { 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_slow": { 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_bytes_wal": { 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "read_random_disk_count": { 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_total_bytes": { 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "slow_used_bytes": { 2024-09-21T09:51:13.642 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_total_bytes": { 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "wal_used_bytes": { 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "write_bytes": { 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2166784 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore": { 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "alloc_unit": { 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4096 2024-09-21T09:51:13.643 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": { 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1290240 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "clist_lat": { 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 3, 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 145763 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_lat": { 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed": { 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_allocated": { 2024-09-21T09:51:13.644 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "compressed_original": { 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "csum_lat": { 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 11, 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 37290 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "decompress_lat": { 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_commit_lat": { 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 51087022 2024-09-21T09:51:13.645 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_final_lat": { 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 181, 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10842897 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_flush_lat": { 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 141645 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "kv_sync_lat": { 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 187, 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 51228667 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_keys_lat": { 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.646 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_get_values_lat": { 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 32222 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_lower_bound_lat": { 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1519 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_next_lat": { 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.647 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_seek_to_first_lat": { 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 0, 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_upper_bound_lat": { 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 2, 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 6757 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_hits": { 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 678 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.648 INFO:tasks.workunit.client.0.smithi033.stdout: "onode_misses": { 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 271 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: "read_lat": { 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 124, 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3240779 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: "read_onode_meta_lat": { 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 202, 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 250409 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.649 INFO:tasks.workunit.client.0.smithi033.stdout: "read_wait_aio_lat": { 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 78, 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2527664 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "reads_with_retries": { 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "remove_lat": { 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 1, 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 15878 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "state_aio_wait_lat": { 2024-09-21T09:51:13.650 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1947884 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_aio_wait_lat": { 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 24, 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1829402 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_cleanup_lat": { 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 24, 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305046592907 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "state_deferred_queued_lat": { 2024-09-21T09:51:13.651 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 24, 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 38336156771 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "state_done_lat": { 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 569025 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "state_finishing_lat": { 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 33441 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "state_io_done_lat": { 2024-09-21T09:51:13.652 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 227813 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_commiting_lat": { 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 144886837 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_done_lat": { 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 97203 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "state_kv_queued_lat": { 2024-09-21T09:51:13.653 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 99237150 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "state_prepare_lat": { 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 17541818 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": { 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 900620 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "truncate_lat": { 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 23, 2024-09-21T09:51:13.654 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 103860 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_commit_lat": { 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 263886316 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_submit_lat": { 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 21332868 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: "txc_throttle_lat": { 2024-09-21T09:51:13.655 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 305, 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 4398408 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache": { 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_bytes": { 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1861455963 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: "heap_bytes": { 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 270974976 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: "mapped_bytes": { 2024-09-21T09:51:13.656 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 269901824 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: "target_bytes": { 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3137367449 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: "unmapped_bytes": { 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 1073152 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:data": { 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.657 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 159383552 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 85481115 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.658 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 155648 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.659 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 126976 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.660 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 73619813 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv": { 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 780140544 2024-09-21T09:51:13.661 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641108365 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.662 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2128 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.663 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71921187 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:kv_onode": { 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 192937984 2024-09-21T09:51:13.664 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 67108864 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 464 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 56987410 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.665 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.666 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 68841246 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "bluestore-pricache:meta": { 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "committed_bytes": { 2024-09-21T09:51:13.667 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 713031680 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "pri0_bytes": { 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "pri10_bytes": { 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "pri11_bytes": { 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 641108365 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "pri1_bytes": { 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "pri2_bytes": { 2024-09-21T09:51:13.668 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 17100 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "pri3_bytes": { 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "pri4_bytes": { 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "pri5_bytes": { 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "pri6_bytes": { 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 260775 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.669 INFO:tasks.workunit.client.0.smithi033.stdout: "pri7_bytes": { 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "pri8_bytes": { 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "pri9_bytes": { 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "reserved_bytes": { 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 71645440 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "objecter": { 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "op_active": { 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.670 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rmw": { 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "osd": { 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg": { 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 46 2024-09-21T09:51:13.671 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "numpg_removing": { 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "op": { 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 129 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_degraded": { 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "op_delayed_unreadable": { 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "op_in_bytes": { 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.672 INFO:tasks.workunit.client.0.smithi033.stdout: "op_latency": { 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 129, 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 82691313 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "op_out_bytes": { 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 265421 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "op_prepare_latency": { 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 131, 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 29586844 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "op_process_latency": { 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 129, 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 60007948 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.673 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r": { 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 90 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency": { 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 13803239 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_latency_out_bytes_histogram": { 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_out_bytes": { 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 265421 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_prepare_latency": { 2024-09-21T09:51:13.674 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 12290984 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "op_r_process_latency": { 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 90, 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 10123032 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw": { 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 34 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_in_bytes": { 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency": { 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 34, 2024-09-21T09:51:13.675 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 32526530 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_in_bytes_histogram": { 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_latency_out_bytes_histogram": { 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_out_bytes": { 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_prepare_latency": { 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 34, 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 14294745 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.676 INFO:tasks.workunit.client.0.smithi033.stdout: "op_rw_process_latency": { 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 34, 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 30198475 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w": { 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 5 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_in_bytes": { 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency": { 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 36361544 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_latency_in_bytes_histogram": { 2024-09-21T09:51:13.677 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_prepare_latency": { 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 7, 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 3001115 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "op_w_process_latency": { 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "count": 5, 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 19686441 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "op_wip": { 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 0 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_bytes": { 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 590368 2024-09-21T09:51:13.678 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: "recovery_ops": { 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 2 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes": { 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 95995035648 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_bytes_used": { 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: "value": 305778688 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: "pools": [ 2024-09-21T09:51:13.679 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr" 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 1, 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 1, 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 1, 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115999498, 2024-09-21T09:51:13.680 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 1781760, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 1781760, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242372001792, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 2, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 2.450442025292432e-06, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 380, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 582656, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 590368, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 590368, 2024-09-21T09:51:13.681 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 1771104, 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 169, 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 1667072 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:13.682 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 32, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 32, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 2, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115999498, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 98304, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 98304, 2024-09-21T09:51:13.683 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242372001792, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 22, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 1.3519711217213626e-07, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 3410, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 3410, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 10230, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 51, 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 19456 2024-09-21T09:51:13.684 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:13.685 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 64, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 64, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 3, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115999498, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 0, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 0, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242372001792, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 0, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:13.686 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 0.0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 0 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:13.687 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd" 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "cache_mode": "none", 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "erasure_code_profile": {}, 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "min_size": 2, 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "options": {}, 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_autoscale_mode": "off", 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 8, 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "pgp_num": 8, 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "pool": 4, 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "size": 3, 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "stats": { 2024-09-21T09:51:13.688 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_raw": 727115999498, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "bytes_used": 12288, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_bytes_used": 0, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "compress_under_bytes": 0, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "data_bytes_used": 12288, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty": 0, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "max_avail": 242372001792, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "objects": 2, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_bytes_used": 0, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "percent_used": 1.689964079787387e-08, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_bytes": 0, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "quota_objects": 0, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "rd": 0, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "rd_bytes": 0, 2024-09-21T09:51:13.689 INFO:tasks.workunit.client.0.smithi033.stdout: "stored": 19, 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_data": 19, 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_omap": 0, 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "stored_raw": 57, 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "wr": 2, 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "wr_bytes": 2048 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_bytes": 0, 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "target_max_objects": 0, 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "type": "replicated" 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd": { 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: "mirroring_by_pool": [ 2024-09-21T09:51:13.690 INFO:tasks.workunit.client.0.smithi033.stdout: false 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "num_images_by_pool": [ 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pools": 1 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "report_id": "baa922a1-6d4a-42b6-88d5-8a719a132b6b", 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "report_timestamp": "2024-09-21T09:51:12.449820", 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "report_version": 1, 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "rocksdb_stats": { 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "6.15.5" 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "rook": { 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "cluster": { 2024-09-21T09:51:13.691 INFO:tasks.workunit.client.0.smithi033.stdout: "external-mode": null, 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "mon": { 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "allow-multiple-per-node": null, 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "count": null, 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "max-id": null, 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "pvc": { 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "enabled": null 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "stretch": { 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "enabled": null 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "network": { 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "provider": null 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.692 INFO:tasks.workunit.client.0.smithi033.stdout: "storage": { 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "device-set": { 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "non-portable": null, 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "portable": null, 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "total": null 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "csi": { 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "kubernetes": { 2024-09-21T09:51:13.693 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "node": { 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "kubernetes-total": null, 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "with-ceph-daemons": null, 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-cephfs-plugin": null, 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-nfs-plugin": null, 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "with-csi-rbd-plugin": null 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "usage": { 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "storage-class": { 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "count": { 2024-09-21T09:51:13.694 INFO:tasks.workunit.client.0.smithi033.stdout: "bucket": null, 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs": null, 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: "nfs": null, 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd": null, 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: "total": null 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: "version": null 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: "services": {}, 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_per_pg": [ 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.695 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.696 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.066672+0000", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.066672+0000", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.066961+0000", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:13.697 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978641+0000", 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.698 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3f", 2024-09-21T09:51:13.699 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:53:59.274014+0000", 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.700 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.701 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.702 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.703 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:13.704 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058702+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058702+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.059165+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.705 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861122+0000", 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.706 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3e", 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:15:58.601280+0000", 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.707 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.708 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.709 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.710 INFO:tasks.workunit.client.0.smithi033.stderr:+ echo OK 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 25 op/s 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='client.24797 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.711 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.712 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.713 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:13.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:12 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:13.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.718 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.719 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060088+0000", 2024-09-21T09:51:13.720 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060088+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060359+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887263+0000", 2024-09-21T09:51:13.721 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3d", 2024-09-21T09:51:13.722 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:30:28.442435+0000", 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.723 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.724 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.725 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:13.726 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.727 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.064743+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.064743+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.065047+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:13.728 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865595+0000", 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.729 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3c", 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:35:11.128919+0000", 2024-09-21T09:51:13.730 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.731 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.732 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.733 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.734 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.735 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:13.736 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.069090+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.069090+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069692+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.737 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978224+0000", 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.738 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3b", 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:50:39.835481+0000", 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.739 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.740 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.741 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.742 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.743 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.744 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.744 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.744 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.744 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:13.744 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.053163+0000", 2024-09-21T09:51:13.745 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.053163+0000", 2024-09-21T09:51:13.745 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.053540+0000", 2024-09-21T09:51:13.745 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:13.745 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865499+0000", 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.746 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3a", 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.747 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:15:48.059464+0000", 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.748 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.749 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.750 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.751 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.752 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.774445+0000", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.774445+0000", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.774760+0000", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:13.753 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978070+0000", 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.754 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.39", 2024-09-21T09:51:13.755 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 31, 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:14:41.770874+0000", 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.756 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.757 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.758 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.759 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.760 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.761 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.762 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.063698+0000", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.063698+0000", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.064091+0000", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:13.763 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079815+0000", 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.764 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.38", 2024-09-21T09:51:13.765 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:26:29.522338+0000", 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.766 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.767 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.768 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.769 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.770 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.771 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.071475+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.071475+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.071747+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:13.772 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978791+0000", 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.773 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.37", 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:09:51.677068+0000", 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.774 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.775 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.776 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.777 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.778 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057940+0000", 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057940+0000", 2024-09-21T09:51:13.779 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058216+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080913+0000", 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.780 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.36", 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.781 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:46:25.737084+0000", 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.782 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.783 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.784 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.785 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.053264+0000", 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.053264+0000", 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.053543+0000", 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:13.786 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080816+0000", 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.787 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.35", 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.788 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:07:31.024310+0000", 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.789 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.790 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.791 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.792 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.850604+0000", 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.850604+0000", 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.850968+0000", 2024-09-21T09:51:13.793 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861618+0000", 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.794 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.34", 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.795 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 18, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:43:37.132880+0000", 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.796 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.797 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.798 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.799 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.064063+0000", 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.064063+0000", 2024-09-21T09:51:13.800 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.064334+0000", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.801 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080096+0000", 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.802 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.33", 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.803 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:01:14.117397+0000", 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.804 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.805 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.806 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.807 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.808 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058245+0000", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058245+0000", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058842+0000", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:13.809 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886822+0000", 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.810 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.32", 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:47:06.517202+0000", 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.811 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.812 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.813 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.814 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.815 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.051766+0000", 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.051766+0000", 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.052028+0000", 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.816 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860590+0000", 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.817 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.31", 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:31:34.964491+0000", 2024-09-21T09:51:13.818 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.819 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.820 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.821 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.822 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.823 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.059852+0000", 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.059852+0000", 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060117+0000", 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:13.824 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865236+0000", 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.825 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.30", 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.826 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:17:51.722984+0000", 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.827 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.828 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.829 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.830 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:13.831 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.480103+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.480103+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.480400+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:13.832 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865999+0000", 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.833 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2f", 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 31, 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:17:34.633257+0000", 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.834 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.835 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.836 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.837 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.838 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:13.839 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.056798+0000", 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.056798+0000", 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.057147+0000", 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.840 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.862210+0000", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.841 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2e", 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.842 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:21:41.842697+0000", 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.843 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.844 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.845 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.846 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.847 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.070359+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.070359+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.070640+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.848 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978661+0000", 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.849 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2d", 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:38:03.602115+0000", 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.850 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.851 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.852 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.853 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.854 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.056360+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.056360+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.056659+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:13.855 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080608+0000", 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.856 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2c", 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:51:06.499056+0000", 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.857 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.858 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.859 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.860 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:13.861 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.064231+0000", 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.064231+0000", 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.065154+0000", 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.862 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886909+0000", 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.863 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2b", 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:17:30.824418+0000", 2024-09-21T09:51:13.864 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.865 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.866 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.867 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.868 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.062437+0000", 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.062437+0000", 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.062716+0000", 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.869 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865558+0000", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.870 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2a", 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.871 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:59:19.327829+0000", 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.872 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.873 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.874 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.875 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.876 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.051090+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.051090+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.051387+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.877 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079839+0000", 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.878 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.29", 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:25:39.466632+0000", 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.879 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.880 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.881 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.882 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:13.883 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057753+0000", 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057753+0000", 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058079+0000", 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.884 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865298+0000", 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.885 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.28", 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:44:44.363386+0000", 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.886 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.887 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.888 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.889 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.890 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.056915+0000", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.056915+0000", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.057220+0000", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:13.891 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080292+0000", 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.892 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.27", 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:52:29.668359+0000", 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.893 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.894 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.895 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.896 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.897 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057047+0000", 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057047+0000", 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.057310+0000", 2024-09-21T09:51:13.898 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080917+0000", 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.899 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.26", 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.900 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:59:34.930735+0000", 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.901 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.902 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.903 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.904 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060725+0000", 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060725+0000", 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060976+0000", 2024-09-21T09:51:13.905 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861084+0000", 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.906 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.25", 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.907 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:20:27.809960+0000", 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.908 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.909 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.910 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.911 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.052853+0000", 2024-09-21T09:51:13.912 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.052853+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.053563+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080326+0000", 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.913 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.24", 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.914 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:51:22.821873+0000", 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.915 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.916 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.917 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.918 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.067911+0000", 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.067911+0000", 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.068204+0000", 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.919 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886944+0000", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.920 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.23", 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:47:42.085769+0000", 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.921 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.922 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.923 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.924 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:13.925 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058587+0000", 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058587+0000", 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058882+0000", 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.926 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865668+0000", 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.927 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.22", 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:06:34.733551+0000", 2024-09-21T09:51:13.928 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.929 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.930 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.931 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:13.932 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060165+0000", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060165+0000", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060446+0000", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:13.933 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861701+0000", 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.934 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.21", 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:39:43.621374+0000", 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.935 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.936 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.937 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.938 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.939 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.059270+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.059270+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.059974+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.940 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080189+0000", 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.941 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.20", 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:56:19.269424+0000", 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.942 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.943 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.944 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.945 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.946 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.048631+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.048631+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.048941+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:13.947 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.859975+0000", 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.948 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1a", 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:59:42.448740+0000", 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.949 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.950 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.951 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.952 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:13.953 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.865732+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.865732+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.866010+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.954 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080417+0000", 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.955 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1b", 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:27:00.193526+0000", 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.956 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.957 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.958 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.959 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.960 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.067412+0000", 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.067412+0000", 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.067813+0000", 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.961 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.962 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1b", 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.963 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:14:17.877346+0000", 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.964 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.965 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.966 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.967 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.480905+0000", 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.480905+0000", 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.481320+0000", 2024-09-21T09:51:13.968 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080597+0000", 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.969 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1a", 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.970 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 35, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:58:23.831301+0000", 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.971 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.972 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.973 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.974 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.859507+0000", 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.859507+0000", 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.859829+0000", 2024-09-21T09:51:13.975 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079961+0000", 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.976 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.19", 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.977 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:01:14.632149+0000", 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.978 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.979 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.980 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.981 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:13.982 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.053295+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.053295+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.053702+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:13.983 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079902+0000", 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.18", 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.984 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:58:24.701874+0000", 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.985 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.986 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.987 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.988 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:13.989 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.862895+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.862895+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.863181+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:13.990 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865313+0000", 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.991 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.18", 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:39:47.364144+0000", 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.992 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:13.993 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:13.994 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:13.995 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:13.996 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.050918+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.050918+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.051248+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:13.997 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860485+0000", 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:13.998 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.19", 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:31:51.653649+0000", 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:13.999 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.000 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.001 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.002 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.865103+0000", 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.865103+0000", 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.865532+0000", 2024-09-21T09:51:14.003 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080973+0000", 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.004 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.17", 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.005 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:58:46.391080+0000", 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.006 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.007 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.008 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.009 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.066939+0000", 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.066939+0000", 2024-09-21T09:51:14.010 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.067330+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865804+0000", 2024-09-21T09:51:14.011 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.16", 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.012 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:07:51.618165+0000", 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.013 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.014 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.015 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.016 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.860288+0000", 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.860288+0000", 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.860605+0000", 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.017 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865951+0000", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.018 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.16", 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.019 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:02:41.592294+0000", 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.020 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.021 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.022 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.023 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.024 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.024 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.024 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.024 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.025 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.025 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.025 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.025 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.025 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057594+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057594+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.057860+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:14.026 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887587+0000", 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.027 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.17", 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:35:19.892986+0000", 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.028 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.029 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.030 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.031 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.856719+0000", 2024-09-21T09:51:14.032 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.856719+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.857026+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860960+0000", 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.033 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 3, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 3, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.15", 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.034 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 27, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T17:17:35.315306+0000", 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 90, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.035 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.036 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 4, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.037 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'3" 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.038 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.059264+0000", 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.059264+0000", 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.059537+0000", 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.039 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860234+0000", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.040 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.14", 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.041 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:05:56.513329+0000", 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.042 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.043 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.044 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.045 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.864160+0000", 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.864160+0000", 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.864446+0000", 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.046 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:49:59.364215+0000", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 12, 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.047 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 12, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.14", 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 36, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:03:53.660368+0000", 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.048 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 2624, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.049 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 12, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 4, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 4, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.050 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 12, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 9 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.051 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "67'12" 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.052 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.069698+0000", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.069698+0000", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.070010+0000", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:14.053 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978421+0000", 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.054 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.15", 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:25:30.278614+0000", 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.055 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.056 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.057 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.058 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.059 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.857951+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.857951+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.858242+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:14.060 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887044+0000", 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.061 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.13", 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:41:48.655094+0000", 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 34, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.062 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:14.063 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.064 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 6 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.065 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.066 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.063116+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.063116+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.063666+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.067 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886995+0000", 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.068 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.12", 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:30:02.411823+0000", 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.069 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.070 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.071 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.072 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.073 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.866875+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.866875+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.867134+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.074 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865774+0000", 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.075 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.12", 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:22:01.081856+0000", 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.076 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.077 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.078 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.079 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.080 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.069569+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.069569+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069831+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.081 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978277+0000", 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.082 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.13", 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:16:52.288583+0000", 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.083 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.084 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.085 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.086 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.087 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.862582+0000", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.862582+0000", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862992+0000", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:14.088 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080355+0000", 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.089 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.11", 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:51:07.188828+0000", 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.090 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.091 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.092 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.093 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.094 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057961+0000", 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057961+0000", 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058311+0000", 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.095 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080230+0000", 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.096 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.10", 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:40:27.743183+0000", 2024-09-21T09:51:14.097 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.098 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.099 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.100 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.101 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.856229+0000", 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.856229+0000", 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.856509+0000", 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:14.102 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860182+0000", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 8, 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.103 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 8, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.10", 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 32, 2024-09-21T09:51:14.104 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:18:23.333228+0000", 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.105 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 12, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 4, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 4, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.106 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 4, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 12, 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.107 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'8" 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.108 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.068917+0000", 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.068917+0000", 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069301+0000", 2024-09-21T09:51:14.109 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978178+0000", 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.110 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.11", 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.111 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T09:49:22.396861+0000", 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.112 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.113 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.114 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.115 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.116 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861815+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861815+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862118+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.117 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860611+0000", 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.118 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.f", 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:16:05.064436+0000", 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 22, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.119 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.120 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.121 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 1, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.122 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.123 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.069235+0000", 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.069235+0000", 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069502+0000", 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.124 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978644+0000", 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.125 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.e", 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:57:08.241392+0000", 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.126 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.127 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.128 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.129 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.130 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.862237+0000", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.862237+0000", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862503+0000", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:14.131 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.862390+0000", 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.132 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.e", 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:44:26.895976+0000", 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.133 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.134 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.135 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.136 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.137 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.068990+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.068990+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.069273+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:14.138 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978686+0000", 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.139 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.f", 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T15:37:10.272940+0000", 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.140 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.141 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.142 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.143 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.144 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.857760+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.857760+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.858095+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.145 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861047+0000", 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.146 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.d", 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:00:57.949722+0000", 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 70, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.147 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.148 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.149 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 1, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.150 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.151 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.063070+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.063070+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.063342+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.152 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865642+0000", 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.153 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.c", 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:48:14.231587+0000", 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.154 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.155 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.156 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.157 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.158 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.859443+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.859443+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.859702+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:14.159 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861853+0000", 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.160 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.c", 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:28:08.636208+0000", 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.161 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.162 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.163 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.164 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.165 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.070074+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.070074+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.070338+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:14.166 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978522+0000", 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.167 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.d", 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:42:32.425964+0000", 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.168 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.169 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.170 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.171 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.172 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.296475+0000", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.296475+0000", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:38.081026+0000", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:14.173 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.656763+0000", 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.174 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.5", 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:24:15.020915+0000", 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000462138, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.175 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.176 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.177 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.178 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.179 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 19, 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:51:09.926291+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:46:57.418997+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:46:57.418997+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:46:57.426595+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:51:09.926291+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:44:57.819716+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:44:57.819716+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 50, 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:51:09.926291+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:51:09.926291+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:51:09.926291+0000", 2024-09-21T09:51:14.180 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:44:57.819716+0000", 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:51:09.926291+0000", 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:51:09.926291+0000", 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 107, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 49, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 107, 2024-09-21T09:51:14.181 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "1.0", 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 158, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:24:39.255282+0000", 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 590368, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.182 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 590368, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:14.183 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 2, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 380, 2024-09-21T09:51:14.184 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 569, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 169, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1628 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "67'107" 2024-09-21T09:51:14.185 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.186 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.866224+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.866224+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.866500+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.187 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.3", 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.188 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:34:25.867104+0000", 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.189 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.190 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.191 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.192 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060632+0000", 2024-09-21T09:51:14.193 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060632+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.061053+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079954+0000", 2024-09-21T09:51:14.194 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.2", 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.195 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:57:34.358891+0000", 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.196 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.197 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.198 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.199 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.298887+0000", 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.298887+0000", 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.887537+0000", 2024-09-21T09:51:14.200 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.509038+0000", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.201 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.6", 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.202 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:51:16.078381+0000", 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000277308, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.203 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.204 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.205 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.206 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.868800+0000", 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.868800+0000", 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.869192+0000", 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.207 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978152+0000", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.208 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.0", 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.209 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:09:11.140972+0000", 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.210 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.211 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.212 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.213 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.056366+0000", 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.056366+0000", 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.056658+0000", 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.214 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.886738+0000", 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.215 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1", 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:01:07.334713+0000", 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.216 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.217 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.218 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.219 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:14.220 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.677383+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.677383+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.861462+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:14.221 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.427162+0000", 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.222 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.7", 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:37:12.106483+0000", 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000471278, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.223 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.224 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.225 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.226 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.227 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861684+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861684+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.861899+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.228 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861820+0000", 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.229 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1", 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:46:38.565876+0000", 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.230 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.231 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.232 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.232 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.232 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.233 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.233 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.233 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.233 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.233 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.233 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.234 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.235 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.057972+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.057972+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058235+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:14.236 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860233+0000", 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.237 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.0", 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:01:01.930427+0000", 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.238 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.239 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.240 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.241 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.242 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.295964+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.295964+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.861803+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:14.243 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.427876+0000", 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.244 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.4", 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:57:02.417883+0000", 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.001053669, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.245 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.246 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.247 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.248 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.249 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861254+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861254+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.861547+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.250 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865760+0000", 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.251 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.2", 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:45:03.154725+0000", 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.252 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.253 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.254 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:14.255 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:14.256 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.059929+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.059929+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060191+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:14.257 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860142+0000", 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.3", 2024-09-21T09:51:14.258 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:32:06.293922+0000", 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.259 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.260 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.261 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.262 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.677309+0000", 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.677309+0000", 2024-09-21T09:51:14.263 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.861137+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.427101+0000", 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.264 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.2", 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.265 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:21:29.622677+0000", 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000569086, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 19, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.266 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.267 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.268 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 2, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 2 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "66'2" 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.269 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861903+0000", 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861903+0000", 2024-09-21T09:51:14.270 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862181+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861250+0000", 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.271 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.4", 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.272 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:26:27.758499+0000", 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.273 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.274 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.275 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.276 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058564+0000", 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058564+0000", 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058849+0000", 2024-09-21T09:51:14.277 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865932+0000", 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.278 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.5", 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.279 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:20:45.591323+0000", 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.280 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.281 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.282 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.283 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.677589+0000", 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.677589+0000", 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.887211+0000", 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.284 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.508707+0000", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.285 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.3", 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 20, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.286 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:59:20.123742+0000", 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000403144, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.287 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.288 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.289 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.290 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "64'1" 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.291 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.866436+0000", 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.866436+0000", 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.866885+0000", 2024-09-21T09:51:14.292 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978674+0000", 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.293 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.5", 2024-09-21T09:51:14.294 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:53:15.223436+0000", 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.295 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.296 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.297 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.298 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.299 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:14.300 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060083+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060083+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060367+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.301 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861149+0000", 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.302 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.4", 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:53:45.400372+0000", 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.303 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.304 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.305 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.306 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.307 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.308 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.298926+0000", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.298926+0000", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:38.080316+0000", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:14.309 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.655198+0000", 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:14.310 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.0", 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.311 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:41:34.343076+0000", 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000557536, 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.312 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.313 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.314 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.315 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.316 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:14.317 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.858680+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.858680+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.858947+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.318 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861460+0000", 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.319 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.6", 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:26:18.567343+0000", 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.320 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.321 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.322 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.323 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 1, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.324 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.325 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060179+0000", 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060179+0000", 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060498+0000", 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.326 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080299+0000", 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.327 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.7", 2024-09-21T09:51:14.328 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:49:25.088288+0000", 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.329 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.330 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.331 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.332 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:14.333 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 61, 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:48:33.677625+0000", 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:48:33.677625+0000", 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:48:37.860425+0000", 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.334 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 62, 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:48:32.280781+0000", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:41.655530+0000", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.335 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 61, 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "4.1", 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.336 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 67, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 19, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:07:51.936200+0000", 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.000698406, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.337 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.338 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.339 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.340 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.341 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.865466+0000", 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.865466+0000", 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.865797+0000", 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.342 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.081020+0000", 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:14.343 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.7", 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.344 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:19:14.491934+0000", 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.345 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.346 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.347 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.348 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.349 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058371+0000", 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058371+0000", 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058690+0000", 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.350 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887191+0000", 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.351 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.6", 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.352 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T13:30:43.395842+0000", 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.353 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.354 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.355 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.356 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.357 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:14.358 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.868686+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.868686+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.868994+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.359 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978109+0000", 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.360 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.8", 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.361 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:44:12.609211+0000", 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.362 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.363 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.364 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.365 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 4, 2024-09-21T09:51:14.366 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.054785+0000", 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.054785+0000", 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.055068+0000", 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.367 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860137+0000", 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.368 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.9", 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.369 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:11:25.336461+0000", 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.370 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.371 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.372 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.373 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 4, 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.374 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.861988+0000", 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.861988+0000", 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.862674+0000", 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.375 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860615+0000", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.376 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.9", 2024-09-21T09:51:14.377 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T20:59:48.674555+0000", 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.378 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.379 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.380 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.381 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: 3 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.382 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.058565+0000", 2024-09-21T09:51:14.383 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.058565+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.058826+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.384 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.079906+0000", 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.385 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.8", 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:11:44.674765+0000", 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.386 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.387 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.388 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.389 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.390 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 1, 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.391 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.860916+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.860916+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.861142+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:14.392 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.860854+0000", 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.393 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.a", 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 24, 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:04:29.958725+0000", 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.394 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.395 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.396 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.397 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: 1, 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: 7 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 1, 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.398 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.399 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.481006+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.481006+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.481406+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.400 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080038+0000", 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.401 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.b", 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 31, 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:18:56.780478+0000", 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.402 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.403 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.404 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.405 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.406 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.407 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.863790+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.863790+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.865234+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:14.408 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978483+0000", 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.409 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.b", 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T11:18:53.965503+0000", 2024-09-21T09:51:14.410 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.411 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.412 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.413 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 1, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.414 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 6, 2024-09-21T09:51:14.415 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.055928+0000", 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.055928+0000", 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.056216+0000", 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.416 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080437+0000", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.417 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.a", 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.418 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:18:05.019873+0000", 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.419 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.420 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.421 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.422 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 6, 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.423 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 5, 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.063293+0000", 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.063293+0000", 2024-09-21T09:51:14.424 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.063533+0000", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.425 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.865838+0000", 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.426 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1d", 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T10:39:49.504603+0000", 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.427 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.428 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.429 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.430 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: 6 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 5, 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.431 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:14.432 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.864212+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.864212+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.864537+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.433 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978478+0000", 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.434 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1c", 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.435 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T21:03:07.329623+0000", 2024-09-21T09:51:14.437 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.437 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.437 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.437 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.438 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.439 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.440 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.441 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 2, 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.442 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:46.773846+0000", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:46.773846+0000", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:46.774197+0000", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 59, 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:14.443 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.861706+0000", 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 58, 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.444 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1c", 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 18, 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.445 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T18:13:13.541469+0000", 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.446 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.447 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.448 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: 2, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: 4, 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: 1 2024-09-21T09:51:14.449 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 2, 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 7, 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.450 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.866026+0000", 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.866026+0000", 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.866709+0000", 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.451 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.978531+0000", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 2, 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.452 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 2, 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1d", 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.453 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 26, 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:28:15.200964+0000", 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.454 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.455 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 3, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 1, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 1, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.456 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 3, 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.457 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: 0 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 7, 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'2" 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.458 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:14.459 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.060536+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.060536+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.060801+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.460 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887503+0000", 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.461 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1f", 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.462 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T14:47:44.619924+0000", 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.463 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.464 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.465 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.466 INFO:tasks.workunit.client.0.smithi033.stdout: 5, 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:14.467 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.864185+0000", 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.864185+0000", 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.864529+0000", 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.468 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080278+0000", 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 1, 2024-09-21T09:51:14.469 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 1, 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1e", 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.470 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 25, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T16:57:11.135766+0000", 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.471 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.472 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.473 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.474 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: 5 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'1" 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.475 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 3, 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 54, 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:24.065373+0000", 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:24.065373+0000", 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:24.065677+0000", 2024-09-21T09:51:14.476 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 55, 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:23.027120+0000", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:14.477 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:38.080170+0000", 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 54, 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.478 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "3.1e", 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 21, 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T12:54:56.431526+0000", 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.479 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.480 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.481 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.482 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: 3, 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: 6, 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: 2 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.483 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 3, 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "0'0" 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": [ 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: "acting_primary": 0, 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: "avail_no_missing": [], 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: "blocked_by": [], 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: "created": 51, 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: "dirty_stats_invalid": false, 2024-09-21T09:51:14.484 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_bytes_stats_invalid": false, 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "hitset_stats_invalid": false, 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_active": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_active": "2024-09-21T09:47:20.865683+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_became_peered": "2024-09-21T09:47:20.865683+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_change": "2024-09-21T09:47:20.865947+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_clean_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub": "0'0", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_deep_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_epoch_clean": 52, 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fresh": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_fullsized": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:14.485 INFO:tasks.workunit.client.0.smithi033.stdout: "last_peered": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub": "0'0", 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_duration": 0, 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "last_scrub_stamp": "2024-09-21T09:47:19.840262+0000", 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "last_undegraded": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "last_unstale": "2024-09-21T09:48:37.887795+0000", 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "log_dups_size": 0, 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 4, 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "log_start": "0'0", 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "manifest_stats_invalid": false, 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "mapping_epoch": 51, 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "object_location_counts": [], 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_scrubbed": 0, 2024-09-21T09:51:14.486 INFO:tasks.workunit.client.0.smithi033.stdout: "objects_trimmed": 0, 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_stats_invalid": false, 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 4, 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_start": "0'0", 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "parent": "0.0", 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "parent_split_bits": 0, 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "pgid": "2.1f", 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "pin_stats_invalid": false, 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "purged_snaps": [], 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_epoch": 66, 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "reported_seq": 28, 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_duration": 0.0, 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "scrub_schedule": "periodic scrub scheduled @ 2024-09-22T19:39:50.666278+0000", 2024-09-21T09:51:14.487 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrim_duration": 0.0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "snaptrimq_len": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 570, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.488 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.489 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 1, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.490 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 4, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "state": "active+clean", 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_invalid": false, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: "up": [ 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: 0, 2024-09-21T09:51:14.491 INFO:tasks.workunit.client.0.smithi033.stdout: 7, 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: 4 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: "up_primary": 0, 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: "version": "57'4" 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_per_pool": [ 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 24, 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: "rbd" 2024-09-21T09:51:14.492 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 3, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pg": 8, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 8, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 3, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "poolid": 4, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 19, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.493 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.494 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.495 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 2, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 2 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 12288, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:14.496 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 57, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 24 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 192, 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:14.497 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 0, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pg": 64, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 8, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 0, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "poolid": 3, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 0, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.498 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.499 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.500 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 0, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 0 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.501 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 192 2024-09-21T09:51:14.502 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 96, 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "cephfs" 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 45, 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pg": 32, 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 8, 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 45, 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "poolid": 2, 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.503 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 3410, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.504 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 66, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 22, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 22, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 14, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.505 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 51, 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 19 2024-09-21T09:51:14.506 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 98304, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 10230, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 96 2024-09-21T09:51:14.507 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: { 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "acting": 3, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "application": [ 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "mgr" 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "log_size": 107, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "num_pg": 1, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "num_store_stats": 3, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "ondisk_log_size": 107, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "poolid": 1, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "stat_sum": { 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes": 590368, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_hit_set_archive": 0, 2024-09-21T09:51:14.508 INFO:tasks.workunit.client.0.smithi033.stdout: "num_bytes_recovered": 590368, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_deep_scrub_errors": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_kb": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_full": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_evict_mode_some": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_kb": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_high": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_flush_mode_low": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_keys_recovered": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_large_omap_objects": 0, 2024-09-21T09:51:14.509 INFO:tasks.workunit.client.0.smithi033.stdout: "num_legacy_snapsets": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_clones": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_object_copies": 6, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects": 2, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_degraded": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_dirty": 2, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_hit_set_archive": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_manifest": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_misplaced": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_missing_on_primary": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_omap": 0, 2024-09-21T09:51:14.510 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_pinned": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_recovered": 2, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_repaired": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_objects_unfound": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_bytes": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_omap_keys": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_promote": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read": 380, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_read_kb": 569, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_scrub_errors": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_shallow_scrub_errors": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_whiteouts": 0, 2024-09-21T09:51:14.511 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write": 169, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "num_write_kb": 1628 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "store_stats": { 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "allocated": 1781760, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "available": 0, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed": 0, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_allocated": 0, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "data_compressed_original": 0, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "data_stored": 1771104, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "internal_metadata": 0, 2024-09-21T09:51:14.512 INFO:tasks.workunit.client.0.smithi033.stdout: "internally_reserved": 0, 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "omap_allocated": 0, 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "total": 0 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "up": 3 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: ], 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "usage": { 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "pg_num": 105, 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "pools": 4, 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "stats_by_class": { 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "ssd": { 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "total_avail_bytes": 765516988416, 2024-09-21T09:51:14.513 INFO:tasks.workunit.client.0.smithi033.stdout: "total_bytes": 767960285184, 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_bytes": 2443296768, 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_raw_bytes": 2443296768, 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_raw_ratio": 0.0031815406400710344 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: }, 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: "total_avail_bytes": 765516988416, 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: "total_bytes": 767960285184, 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: "total_used_bytes": 2443296768 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout: } 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout:} 2024-09-21T09:51:14.514 INFO:tasks.workunit.client.0.smithi033.stdout:OK 2024-09-21T09:51:14.515 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T09:51:14.515 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 25 op/s 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='client.24797 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.516 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.517 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.518 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.518 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.518 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.518 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.518 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.518 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "telemetry show", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.518 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: pgmap v91: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 25 op/s 2024-09-21T09:51:14.518 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='client.24797 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.519 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.520 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.521 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:13 smithi105 ceph-mon[31286]: from='client.24797 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.521 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:13 smithi105 ceph-mon[31286]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.521 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:13 smithi105 ceph-mon[31286]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.521 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.521 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.521 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.521 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:14.521 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.522 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.523 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.524 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.524 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.524 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.524 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.524 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:14.524 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:51:14.524 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.524 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.525 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:51:14.526 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:12 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:51:14.527 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:13 smithi033 ceph-mon[39045]: from='client.24797 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.527 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:13 smithi033 ceph-mon[39045]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.527 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:13 smithi033 ceph-mon[39045]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.527 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:13 smithi033 ceph-mon[27969]: from='client.24797 -' entity='client.admin' cmd=[{"prefix": "telemetry show-device", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.527 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:13 smithi033 ceph-mon[27969]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.527 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:13 smithi033 ceph-mon[27969]: from='client.14898 -' entity='client.admin' cmd=[{"prefix": "telemetry show-all", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:14.562 INFO:tasks.workunit:Stopping ['test_telemetry_quincy.sh'] on client.0... 2024-09-21T09:51:14.562 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-21T09:51:14.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:14 smithi033 ceph-mon[39045]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 25 op/s 2024-09-21T09:51:14.888 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:14 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:14.888 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:14 smithi033 ceph-mon[27969]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 25 op/s 2024-09-21T09:51:14.888 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:14 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:14.889 DEBUG:teuthology.parallel:result is None 2024-09-21T09:51:14.889 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:51:14.915 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:51:14.915 DEBUG:teuthology.orchestra.run.smithi033:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T09:51:14.979 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T09:51:14.979 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:51:14.988 INFO:teuthology.task.print:**** done end telemetry quincy... 2024-09-21T09:51:14.989 INFO:teuthology.run_tasks:Running task print... 2024-09-21T09:51:15.024 INFO:teuthology.task.print:**** done start parallel 2024-09-21T09:51:15.024 INFO:teuthology.run_tasks:Running task parallel... 2024-09-21T09:51:15.068 INFO:teuthology.task.parallel:starting parallel... 2024-09-21T09:51:15.068 INFO:teuthology.task.parallel:In parallel, running task full_sequential... 2024-09-21T09:51:15.069 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-21T09:51:15.069 INFO:teuthology.task.print:**** done start ec-rados-default.yaml 2024-09-21T09:51:15.069 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2024-09-21T09:51:15.071 INFO:tasks.rados:Beginning rados... 2024-09-21T09:51:15.071 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-21T09:51:15.071 INFO:tasks.rados:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(POOL_APP_NOT_ENABLED\\)', 'OSD_DOWN', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_DEGRADED', 'Reduced data availability', 'Degraded data redundancy', 'OSDMAP_FLAGS', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'extra_system_packages': ['python3-pytest'], 'flavor': 'default', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '62da54b4b83a96ca9976b7e1d0c8272a564a2208'}} 2024-09-21T09:51:15.072 INFO:tasks.rados:config is {'clients': ['client.0'], 'ec_pool': True, 'objects': 50, 'op_weights': {'append': 100, 'copy_from': 50, 'delete': 50, 'read': 100, 'rmattr': 25, 'rollback': 50, 'setattr': 25, 'snap_create': 50, 'snap_remove': 50, 'write': 0}, 'ops': 4000, 'write_append_excl': False} 2024-09-21T09:51:15.072 INFO:tasks.rados:joining rados 2024-09-21T09:51:15.072 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-21T09:51:15.072 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T09:51:15.073 INFO:teuthology.task.print:**** done start upgrade, wait 2024-09-21T09:51:15.073 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-21T09:51:15.073 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi033.front.sepia.ceph.com 2024-09-21T09:51:15.073 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-21T09:51:15.074 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2024-09-21T09:51:15.074 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd erasure-code-profile set teuthologyprofile k=2 m=1 crush-failure-domain=osd 2024-09-21T09:51:15.075 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:14 smithi105 ceph-mon[31286]: pgmap v92: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 0 B/s wr, 25 op/s 2024-09-21T09:51:15.075 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:14 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:15.339 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:15.339 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:16.967 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:16 smithi033 ceph-mon[39045]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-21T09:51:16.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:16 smithi033 ceph-mon[27969]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-21T09:51:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:16 smithi105 ceph-mon[31286]: pgmap v93: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-21T09:51:18.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:18 smithi033 ceph-mon[39045]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-21T09:51:18.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:18 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1237227533' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:51:18.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:18 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:51:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:18 smithi033 ceph-mon[27969]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-21T09:51:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:18 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1237227533' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:51:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:18 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:51:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:18 smithi105 ceph-mon[31286]: pgmap v94: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 44 op/s 2024-09-21T09:51:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:18 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1237227533' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:51:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:18 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]: dispatch 2024-09-21T09:51:19.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:18] "GET /metrics HTTP/1.1" 200 236503 "" "Prometheus/2.43.0" 2024-09-21T09:51:19.520 INFO:tasks.rados:starting run 0 out of 1 2024-09-21T09:51:19.521 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-21T09:51:19.521 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool create unique_pool_0 16 16 erasure teuthologyprofile 2024-09-21T09:51:19.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:19 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-21T09:51:19.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:19 smithi033 ceph-mon[27969]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:51:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:19 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-21T09:51:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:19 smithi033 ceph-mon[39045]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:51:19.894 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-21T09:51:19.975 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:19 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd erasure-code-profile set", "name": "teuthologyprofile", "profile": ["k=2", "m=1", "crush-failure-domain=osd"]}]': finished 2024-09-21T09:51:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:19 smithi105 ceph-mon[31286]: osdmap e68: 8 total, 8 up, 8 in 2024-09-21T09:51:20.713 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:20.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:20 smithi033 ceph-mon[39045]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 0 B/s wr, 52 op/s 2024-09-21T09:51:20.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:20 smithi033 ceph-mon[27969]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 0 B/s wr, 52 op/s 2024-09-21T09:51:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:20 smithi105 ceph-mon[31286]: pgmap v96: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 0 B/s wr, 52 op/s 2024-09-21T09:51:22.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:51:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:51:23.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:23 smithi033 ceph-mon[39045]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 21 op/s 2024-09-21T09:51:23.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:23 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3787492911' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-21T09:51:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:23 smithi033 ceph-mon[27969]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 21 op/s 2024-09-21T09:51:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:23 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3787492911' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-21T09:51:23.383 INFO:teuthology.orchestra.run.smithi033.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:51:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:23 smithi105 ceph-mon[31286]: pgmap v97: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 21 op/s 2024-09-21T09:51:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:23 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3787492911' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]: dispatch 2024-09-21T09:51:23.692 INFO:teuthology.orchestra.run.smithi033.stderr:pool 'unique_pool_0' created 2024-09-21T09:51:23.979 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-21T09:51:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:51:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:51:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3787492911' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[39045]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:24.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:24.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:24.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:51:24.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3787492911' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-21T09:51:24.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:24 smithi033 ceph-mon[27969]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:51:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: osdmap e69: 8 total, 8 up, 8 in 2024-09-21T09:51:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: pgmap v99: 105 pgs: 105 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:51:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:24.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:24.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:51:24.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:51:24.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:24.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 2024-09-21T09:51:24.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3787492911' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16, "pgp_num": 16, "pool_type": "erasure", "erasure_code_profile": "teuthologyprofile"}]': finished 2024-09-21T09:51:24.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:24 smithi105 ceph-mon[31286]: osdmap e70: 8 total, 8 up, 8 in 2024-09-21T09:51:24.823 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-21T09:51:24.879 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:25.692 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:25 smithi033 ceph-mon[27969]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:51:25.743 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T09:51:25.975 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:25 smithi033 ceph-mon[39045]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:51:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:25 smithi105 ceph-mon[31286]: osdmap e71: 8 total, 8 up, 8 in 2024-09-21T09:51:26.693 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:26 smithi033 ceph-mon[27969]: pgmap v102: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:26.693 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:26 smithi033 ceph-mon[27969]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:51:26.967 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:26 smithi033 ceph-mon[39045]: pgmap v102: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:26.967 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:26 smithi033 ceph-mon[39045]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:51:26.967 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:26 smithi033 ceph-mon[39045]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:51:26.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:26 smithi033 ceph-mon[27969]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:51:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:26 smithi105 ceph-mon[31286]: pgmap v102: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:26 smithi105 ceph-mon[31286]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T09:51:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:26 smithi105 ceph-mon[31286]: osdmap e72: 8 total, 8 up, 8 in 2024-09-21T09:51:28.618 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:51:28.700 INFO:teuthology.orchestra.run.smithi033.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-21T09:51:28.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:28 smithi033 ceph-mon[39045]: pgmap v104: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:28.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:28 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1041580246' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:51:28.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:28 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:51:28.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:28 smithi033 ceph-mon[27969]: pgmap v104: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:28.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:28 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1041580246' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:51:28.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:28 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:51:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:28 smithi105 ceph-mon[31286]: pgmap v104: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:28 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1041580246' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:51:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:28 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2024-09-21T09:51:29.245 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:28] "GET /metrics HTTP/1.1" 200 236503 "" "Prometheus/2.43.0" 2024-09-21T09:51:29.248 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:51:29.248 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (106s) 88s ago 2m 17.3M - 0.25.0 c8568f914cd2 f5c871f25675 2024-09-21T09:51:29.248 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (2m) 91s ago 2m 82.5M - 9.4.7 954c08fa6188 39f7f9958856 2024-09-21T09:51:29.248 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (4m) 88s ago 4m 13.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:51:29.248 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (4m) 91s ago 4m 16.8M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:51:29.248 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443 running (7m) 91s ago 7m 399M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be e1080a983fa6 2024-09-21T09:51:29.248 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:9283,8765 running (9m) 88s ago 9m 445M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 21061eedc154 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (9m) 88s ago 9m 48.2M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 220cb0bc67bb 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (8m) 91s ago 8m 40.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c2741af3f853 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (8m) 88s ago 7m 41.4M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f8b68f40cc5 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (2m) 88s ago 2m 13.0M - 1.5.0 0da6a335fe13 ae258c372f47 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (2m) 91s ago 2m 3376k - 1.5.0 0da6a335fe13 446328798f33 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (7m) 88s ago 7m 43.9M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (7m) 88s ago 7m 44.3M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (6m) 88s ago 6m 44.8M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (6m) 88s ago 6m 39.8M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (5m) 91s ago 5m 42.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (5m) 91s ago 5m 42.6M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (5m) 91s ago 5m 42.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (4m) 91s ago 4m 42.4M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:51:29.249 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (101s) 91s ago 3m 24.0M - 2.43.0 a07b618ecd1d 9b3306609c60 2024-09-21T09:51:29.617 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:51:29.617 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:51:29.617 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:51:29.617 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:51:29.617 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:51:29.617 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:51:29.617 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 15 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:51:29.618 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:51:29.929 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:51:29.929 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208", 2024-09-21T09:51:29.930 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:51:29.930 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:51:29.930 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-21T09:51:29.930 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "", 2024-09-21T09:51:29.930 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:62da54b4b83a96ca9976b7e1d0c8272a564a2208 image", 2024-09-21T09:51:29.930 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:51:29.930 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:51:29.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[39045]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:29.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[39045]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:51:29.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[39045]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:51:29.930 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:29.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:29.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/1865433860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:51:29.931 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[27969]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:29.931 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[27969]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:51:29.931 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[27969]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:51:29.931 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:29.931 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:29.931 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:29 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/1865433860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:51:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:29 smithi105 ceph-mon[31286]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:29 smithi105 ceph-mon[31286]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-21T09:51:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:29 smithi105 ceph-mon[31286]: osdmap e73: 8 total, 8 up, 8 in 2024-09-21T09:51:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:30.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:29 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/1865433860' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:51:30.377 DEBUG:teuthology.orchestra.run.smithi033:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --no-omap --ec-pool --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 0 --op delete 50 --op snap_create 50 --op snap_remove 50 --op rollback 50 --op setattr 25 --op rmattr 25 --op copy_from 50 --op append 100 --pool unique_pool_0 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight read -> 100 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight delete -> 50 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight snap_create -> 50 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight snap_remove -> 50 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight rollback -> 50 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight setattr -> 25 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight rmattr -> 25 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight copy_from -> 50 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:adding op weight append -> 100 2024-09-21T09:51:30.405 INFO:tasks.rados.rados.0.smithi033.stdout:ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable) 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi033.stdout:Configuration: 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi033.stdout: Number of operations: 4000 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi033.stdout: Number of objects: 50 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi033.stdout: Max in flight operations: 16 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi033.stdout: Object size (in bytes): 4000000 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi033.stdout: Write stride min: 400000 2024-09-21T09:51:30.406 INFO:tasks.rados.rados.0.smithi033.stdout: Write stride max: 800000 2024-09-21T09:51:30.717 INFO:tasks.rados.rados.0.smithi033.stdout:1: write initial oid 1 2024-09-21T09:51:30.717 INFO:tasks.rados.rados.0.smithi033.stdout:1: seq_num 0 ranges {0=24576} 2024-09-21T09:51:30.717 INFO:tasks.rados.rados.0.smithi033.stdout:1: writing smithi03385152-1 from 0 to 24576 tid 1 2024-09-21T09:51:30.717 INFO:tasks.rados.rados.0.smithi033.stdout:2: write initial oid 2 2024-09-21T09:51:30.717 INFO:tasks.rados.rados.0.smithi033.stdout:2: seq_num 1 ranges {0=24576} 2024-09-21T09:51:30.717 INFO:tasks.rados.rados.0.smithi033.stdout:2: writing smithi03385152-2 from 0 to 24576 tid 1 2024-09-21T09:51:30.717 INFO:tasks.rados.rados.0.smithi033.stdout:3: write initial oid 3 2024-09-21T09:51:30.717 INFO:tasks.rados.rados.0.smithi033.stdout:3: seq_num 2 ranges {0=40960} 2024-09-21T09:51:30.718 INFO:tasks.rados.rados.0.smithi033.stdout:3: writing smithi03385152-3 from 0 to 40960 tid 1 2024-09-21T09:51:30.718 INFO:tasks.rados.rados.0.smithi033.stdout:4: write initial oid 4 2024-09-21T09:51:30.718 INFO:tasks.rados.rados.0.smithi033.stdout:4: seq_num 3 ranges {0=57344} 2024-09-21T09:51:30.718 INFO:tasks.rados.rados.0.smithi033.stdout:4: writing smithi03385152-4 from 0 to 57344 tid 1 2024-09-21T09:51:30.718 INFO:tasks.rados.rados.0.smithi033.stdout:5: write initial oid 5 2024-09-21T09:51:30.718 INFO:tasks.rados.rados.0.smithi033.stdout:5: seq_num 4 ranges {0=73728} 2024-09-21T09:51:30.719 INFO:tasks.rados.rados.0.smithi033.stdout:5: writing smithi03385152-5 from 0 to 73728 tid 1 2024-09-21T09:51:30.719 INFO:tasks.rados.rados.0.smithi033.stdout:6: write initial oid 6 2024-09-21T09:51:30.719 INFO:tasks.rados.rados.0.smithi033.stdout:6: seq_num 5 ranges {0=90112} 2024-09-21T09:51:30.719 INFO:tasks.rados.rados.0.smithi033.stdout:6: writing smithi03385152-6 from 0 to 90112 tid 1 2024-09-21T09:51:30.719 INFO:tasks.rados.rados.0.smithi033.stdout:7: write initial oid 7 2024-09-21T09:51:30.719 INFO:tasks.rados.rados.0.smithi033.stdout:7: seq_num 6 ranges {0=106496} 2024-09-21T09:51:30.720 INFO:tasks.rados.rados.0.smithi033.stdout:7: writing smithi03385152-7 from 0 to 106496 tid 1 2024-09-21T09:51:30.720 INFO:tasks.rados.rados.0.smithi033.stdout:8: write initial oid 8 2024-09-21T09:51:30.720 INFO:tasks.rados.rados.0.smithi033.stdout:8: seq_num 7 ranges {0=122880} 2024-09-21T09:51:30.721 INFO:tasks.rados.rados.0.smithi033.stdout:8: writing smithi03385152-8 from 0 to 122880 tid 1 2024-09-21T09:51:30.722 INFO:tasks.rados.rados.0.smithi033.stdout:9: write initial oid 9 2024-09-21T09:51:30.722 INFO:tasks.rados.rados.0.smithi033.stdout:9: seq_num 8 ranges {0=139264} 2024-09-21T09:51:30.722 INFO:tasks.rados.rados.0.smithi033.stdout:9: writing smithi03385152-9 from 0 to 139264 tid 1 2024-09-21T09:51:30.723 INFO:tasks.rados.rados.0.smithi033.stdout:10: write initial oid 10 2024-09-21T09:51:30.723 INFO:tasks.rados.rados.0.smithi033.stdout:10: seq_num 9 ranges {0=155648} 2024-09-21T09:51:30.724 INFO:tasks.rados.rados.0.smithi033.stdout:10: writing smithi03385152-10 from 0 to 155648 tid 1 2024-09-21T09:51:30.724 INFO:tasks.rados.rados.0.smithi033.stdout:2: finishing write tid 1 to smithi03385152-2 2024-09-21T09:51:30.724 INFO:tasks.rados.rados.0.smithi033.stdout:3: finishing write tid 1 to smithi03385152-3 2024-09-21T09:51:30.724 INFO:tasks.rados.rados.0.smithi033.stdout:2: finishing write tid 2 to smithi03385152-2 2024-09-21T09:51:30.724 INFO:tasks.rados.rados.0.smithi033.stdout:11: write initial oid 11 2024-09-21T09:51:30.725 INFO:tasks.rados.rados.0.smithi033.stdout:11: seq_num 10 ranges {0=172032} 2024-09-21T09:51:30.726 INFO:tasks.rados.rados.0.smithi033.stdout:11: writing smithi03385152-11 from 0 to 172032 tid 1 2024-09-21T09:51:30.726 INFO:tasks.rados.rados.0.smithi033.stdout:12: write initial oid 12 2024-09-21T09:51:30.726 INFO:tasks.rados.rados.0.smithi033.stdout:12: seq_num 11 ranges {0=188416} 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:12: writing smithi03385152-12 from 0 to 188416 tid 1 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:13: write initial oid 13 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:2: finishing write tid 3 to smithi03385152-2 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:5: finishing write tid 1 to smithi03385152-5 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:5: finishing write tid 2 to smithi03385152-5 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:1: finishing write tid 1 to smithi03385152-1 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:1: finishing write tid 2 to smithi03385152-1 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:1: finishing write tid 3 to smithi03385152-1 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 2 (ObjNum 0 snap 0 seq_num 0) dirty exists 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:3: finishing write tid 2 to smithi03385152-3 2024-09-21T09:51:30.728 INFO:tasks.rados.rados.0.smithi033.stdout:4: finishing write tid 1 to smithi03385152-4 2024-09-21T09:51:30.729 INFO:tasks.rados.rados.0.smithi033.stdout:4: finishing write tid 2 to smithi03385152-4 2024-09-21T09:51:30.729 INFO:tasks.rados.rados.0.smithi033.stdout:9: finishing write tid 1 to smithi03385152-9 2024-09-21T09:51:30.729 INFO:tasks.rados.rados.0.smithi033.stdout:6: finishing write tid 1 to smithi03385152-6 2024-09-21T09:51:30.729 INFO:tasks.rados.rados.0.smithi033.stdout:13: seq_num 12 ranges {0=204800} 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:13: writing smithi03385152-13 from 0 to 204800 tid 1 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:9: finishing write tid 2 to smithi03385152-9 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:7: finishing write tid 1 to smithi03385152-7 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:6: finishing write tid 2 to smithi03385152-6 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:8: finishing write tid 1 to smithi03385152-8 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:3: finishing write tid 3 to smithi03385152-3 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2 (ObjNum 2 snap 0 seq_num 2) dirty exists 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:10: finishing write tid 1 to smithi03385152-10 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:5: finishing write tid 3 to smithi03385152-5 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2 (ObjNum 4 snap 0 seq_num 4) dirty exists 2024-09-21T09:51:30.730 INFO:tasks.rados.rados.0.smithi033.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2024-09-21T09:51:30.731 INFO:tasks.rados.rados.0.smithi033.stdout:1: done (12 left) 2024-09-21T09:51:30.731 INFO:tasks.rados.rados.0.smithi033.stdout:2: done (11 left) 2024-09-21T09:51:30.731 INFO:tasks.rados.rados.0.smithi033.stdout:3: done (10 left) 2024-09-21T09:51:30.731 INFO:tasks.rados.rados.0.smithi033.stdout:5: done (9 left) 2024-09-21T09:51:30.731 INFO:tasks.rados.rados.0.smithi033.stdout:14: write initial oid 14 2024-09-21T09:51:30.731 INFO:tasks.rados.rados.0.smithi033.stdout:14: seq_num 13 ranges {0=221184} 2024-09-21T09:51:30.732 INFO:tasks.rados.rados.0.smithi033.stdout:14: writing smithi03385152-14 from 0 to 221184 tid 1 2024-09-21T09:51:30.732 INFO:tasks.rados.rados.0.smithi033.stdout:15: write initial oid 15 2024-09-21T09:51:30.732 INFO:tasks.rados.rados.0.smithi033.stdout:15: seq_num 14 ranges {0=237568} 2024-09-21T09:51:30.734 INFO:tasks.rados.rados.0.smithi033.stdout:15: writing smithi03385152-15 from 0 to 237568 tid 1 2024-09-21T09:51:30.734 INFO:tasks.rados.rados.0.smithi033.stdout:6: finishing write tid 3 to smithi03385152-6 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 2 (ObjNum 5 snap 0 seq_num 5) dirty exists 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:4: finishing write tid 3 to smithi03385152-4 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 2 (ObjNum 3 snap 0 seq_num 3) dirty exists 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:8: finishing write tid 2 to smithi03385152-8 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:10: finishing write tid 2 to smithi03385152-10 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:12: finishing write tid 1 to smithi03385152-12 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:12: finishing write tid 2 to smithi03385152-12 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:9: finishing write tid 3 to smithi03385152-9 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 2 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:10: finishing write tid 3 to smithi03385152-10 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 2 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:7: finishing write tid 2 to smithi03385152-7 2024-09-21T09:51:30.735 INFO:tasks.rados.rados.0.smithi033.stdout:12: finishing write tid 3 to smithi03385152-12 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:8: finishing write tid 3 to smithi03385152-8 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 4 (ObjNum 7 snap 0 seq_num 7) dirty exists 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:11: finishing write tid 1 to smithi03385152-11 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:11: finishing write tid 2 to smithi03385152-11 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:13: finishing write tid 1 to smithi03385152-13 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:7: finishing write tid 3 to smithi03385152-7 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:14: finishing write tid 1 to smithi03385152-14 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:4: done (10 left) 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:6: done (9 left) 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:7: done (8 left) 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:8: done (7 left) 2024-09-21T09:51:30.736 INFO:tasks.rados.rados.0.smithi033.stdout:9: done (6 left) 2024-09-21T09:51:30.737 INFO:tasks.rados.rados.0.smithi033.stdout:10: done (5 left) 2024-09-21T09:51:30.737 INFO:tasks.rados.rados.0.smithi033.stdout:12: done (4 left) 2024-09-21T09:51:30.737 INFO:tasks.rados.rados.0.smithi033.stdout:16: write initial oid 16 2024-09-21T09:51:30.737 INFO:tasks.rados.rados.0.smithi033.stdout:16: seq_num 15 ranges {0=253952} 2024-09-21T09:51:30.737 INFO:tasks.rados.rados.0.smithi033.stdout:16: writing smithi03385152-16 from 0 to 253952 tid 1 2024-09-21T09:51:30.737 INFO:tasks.rados.rados.0.smithi033.stdout:17: write initial oid 17 2024-09-21T09:51:30.738 INFO:tasks.rados.rados.0.smithi033.stdout:17: seq_num 16 ranges {0=270336} 2024-09-21T09:51:30.738 INFO:tasks.rados.rados.0.smithi033.stdout:17: writing smithi03385152-17 from 0 to 270336 tid 1 2024-09-21T09:51:30.738 INFO:tasks.rados.rados.0.smithi033.stdout:13: finishing write tid 2 to smithi03385152-13 2024-09-21T09:51:30.738 INFO:tasks.rados.rados.0.smithi033.stdout:18: write initial oid 18 2024-09-21T09:51:30.738 INFO:tasks.rados.rados.0.smithi033.stdout:18: seq_num 17 ranges {0=286720} 2024-09-21T09:51:30.739 INFO:tasks.rados.rados.0.smithi033.stdout:18: writing smithi03385152-18 from 0 to 286720 tid 1 2024-09-21T09:51:30.739 INFO:tasks.rados.rados.0.smithi033.stdout:19: write initial oid 19 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:11: finishing write tid 3 to smithi03385152-11 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 2 (ObjNum 10 snap 0 seq_num 10) dirty exists 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:14: finishing write tid 2 to smithi03385152-14 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:13: finishing write tid 3 to smithi03385152-13 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 4 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:14: finishing write tid 3 to smithi03385152-14 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 4 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:15: finishing write tid 1 to smithi03385152-15 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:15: finishing write tid 2 to smithi03385152-15 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:16: finishing write tid 1 to smithi03385152-16 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:16: finishing write tid 2 to smithi03385152-16 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:15: finishing write tid 3 to smithi03385152-15 2024-09-21T09:51:30.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 6 (ObjNum 14 snap 0 seq_num 14) dirty exists 2024-09-21T09:51:30.741 INFO:tasks.rados.rados.0.smithi033.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:51:30.741 INFO:tasks.rados.rados.0.smithi033.stdout:19: seq_num 18 ranges {0=303104} 2024-09-21T09:51:30.741 INFO:tasks.rados.rados.0.smithi033.stdout:19: writing smithi03385152-19 from 0 to 303104 tid 1 2024-09-21T09:51:30.741 INFO:tasks.rados.rados.0.smithi033.stdout:11: done (7 left) 2024-09-21T09:51:30.741 INFO:tasks.rados.rados.0.smithi033.stdout:13: done (6 left) 2024-09-21T09:51:30.741 INFO:tasks.rados.rados.0.smithi033.stdout:14: done (5 left) 2024-09-21T09:51:30.741 INFO:tasks.rados.rados.0.smithi033.stdout:15: done (4 left) 2024-09-21T09:51:30.742 INFO:tasks.rados.rados.0.smithi033.stdout:20: write initial oid 20 2024-09-21T09:51:30.742 INFO:tasks.rados.rados.0.smithi033.stdout:20: seq_num 19 ranges {0=319488} 2024-09-21T09:51:30.744 INFO:tasks.rados.rados.0.smithi033.stdout:20: writing smithi03385152-20 from 0 to 319488 tid 1 2024-09-21T09:51:30.744 INFO:tasks.rados.rados.0.smithi033.stdout:17: finishing write tid 1 to smithi03385152-17 2024-09-21T09:51:30.744 INFO:tasks.rados.rados.0.smithi033.stdout:17: finishing write tid 2 to smithi03385152-17 2024-09-21T09:51:30.744 INFO:tasks.rados.rados.0.smithi033.stdout:16: finishing write tid 3 to smithi03385152-16 2024-09-21T09:51:30.744 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 6 (ObjNum 15 snap 0 seq_num 15) dirty exists 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:17: finishing write tid 3 to smithi03385152-17 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 4 (ObjNum 16 snap 0 seq_num 16) dirty exists 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:18: finishing write tid 1 to smithi03385152-18 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:18: finishing write tid 2 to smithi03385152-18 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:16: done (4 left) 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:17: done (3 left) 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:21: write initial oid 21 2024-09-21T09:51:30.745 INFO:tasks.rados.rados.0.smithi033.stdout:21: seq_num 20 ranges {0=344064} 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:21: writing smithi03385152-21 from 0 to 344064 tid 1 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:22: write initial oid 22 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:18: finishing write tid 3 to smithi03385152-18 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 2 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:19: finishing write tid 1 to smithi03385152-19 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:19: finishing write tid 2 to smithi03385152-19 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:19: finishing write tid 3 to smithi03385152-19 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 4 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:20: finishing write tid 1 to smithi03385152-20 2024-09-21T09:51:30.747 INFO:tasks.rados.rados.0.smithi033.stdout:22: seq_num 21 ranges {0=360448} 2024-09-21T09:51:30.749 INFO:tasks.rados.rados.0.smithi033.stdout:22: writing smithi03385152-22 from 0 to 360448 tid 1 2024-09-21T09:51:30.749 INFO:tasks.rados.rados.0.smithi033.stdout:18: done (4 left) 2024-09-21T09:51:30.749 INFO:tasks.rados.rados.0.smithi033.stdout:19: done (3 left) 2024-09-21T09:51:30.749 INFO:tasks.rados.rados.0.smithi033.stdout:23: write initial oid 23 2024-09-21T09:51:30.750 INFO:tasks.rados.rados.0.smithi033.stdout:20: finishing write tid 2 to smithi03385152-20 2024-09-21T09:51:30.750 INFO:tasks.rados.rados.0.smithi033.stdout:21: finishing write tid 1 to smithi03385152-21 2024-09-21T09:51:30.750 INFO:tasks.rados.rados.0.smithi033.stdout:23: seq_num 22 ranges {0=376832} 2024-09-21T09:51:30.751 INFO:tasks.rados.rados.0.smithi033.stdout:23: writing smithi03385152-23 from 0 to 376832 tid 1 2024-09-21T09:51:30.751 INFO:tasks.rados.rados.0.smithi033.stdout:24: write initial oid 24 2024-09-21T09:51:30.751 INFO:tasks.rados.rados.0.smithi033.stdout:24: seq_num 23 ranges {0=393216} 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:24: writing smithi03385152-24 from 0 to 393216 tid 1 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:21: finishing write tid 2 to smithi03385152-21 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:20: finishing write tid 3 to smithi03385152-20 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 2 (ObjNum 19 snap 0 seq_num 19) dirty exists 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:21: finishing write tid 3 to smithi03385152-21 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 4 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:51:30.754 INFO:tasks.rados.rados.0.smithi033.stdout:22: finishing write tid 1 to smithi03385152-22 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:22: finishing write tid 2 to smithi03385152-22 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:23: finishing write tid 1 to smithi03385152-23 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:23: finishing write tid 2 to smithi03385152-23 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:22: finishing write tid 3 to smithi03385152-22 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 6 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:20: done (4 left) 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:21: done (3 left) 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:22: done (2 left) 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:25: write initial oid 25 2024-09-21T09:51:30.755 INFO:tasks.rados.rados.0.smithi033.stdout:25: seq_num 24 ranges {0=409600} 2024-09-21T09:51:30.757 INFO:tasks.rados.rados.0.smithi033.stdout:25: writing smithi03385152-25 from 0 to 409600 tid 1 2024-09-21T09:51:30.757 INFO:tasks.rados.rados.0.smithi033.stdout:23: finishing write tid 3 to smithi03385152-23 2024-09-21T09:51:30.757 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 6 (ObjNum 22 snap 0 seq_num 22) dirty exists 2024-09-21T09:51:30.757 INFO:tasks.rados.rados.0.smithi033.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2024-09-21T09:51:30.757 INFO:tasks.rados.rados.0.smithi033.stdout:24: finishing write tid 1 to smithi03385152-24 2024-09-21T09:51:30.757 INFO:tasks.rados.rados.0.smithi033.stdout:24: finishing write tid 2 to smithi03385152-24 2024-09-21T09:51:30.757 INFO:tasks.rados.rados.0.smithi033.stdout:23: done (2 left) 2024-09-21T09:51:30.758 INFO:tasks.rados.rados.0.smithi033.stdout:26: write initial oid 26 2024-09-21T09:51:30.758 INFO:tasks.rados.rados.0.smithi033.stdout:26: seq_num 25 ranges {0=425984} 2024-09-21T09:51:30.760 INFO:tasks.rados.rados.0.smithi033.stdout:26: writing smithi03385152-26 from 0 to 425984 tid 1 2024-09-21T09:51:30.760 INFO:tasks.rados.rados.0.smithi033.stdout:27: write initial oid 27 2024-09-21T09:51:30.760 INFO:tasks.rados.rados.0.smithi033.stdout:27: seq_num 26 ranges {0=442368} 2024-09-21T09:51:30.763 INFO:tasks.rados.rados.0.smithi033.stdout:27: writing smithi03385152-27 from 0 to 442368 tid 1 2024-09-21T09:51:30.763 INFO:tasks.rados.rados.0.smithi033.stdout:28: write initial oid 28 2024-09-21T09:51:30.763 INFO:tasks.rados.rados.0.smithi033.stdout:28: seq_num 27 ranges {0=458752} 2024-09-21T09:51:30.766 INFO:tasks.rados.rados.0.smithi033.stdout:28: writing smithi03385152-28 from 0 to 458752 tid 1 2024-09-21T09:51:30.766 INFO:tasks.rados.rados.0.smithi033.stdout:29: write initial oid 29 2024-09-21T09:51:30.766 INFO:tasks.rados.rados.0.smithi033.stdout:29: seq_num 28 ranges {0=475136} 2024-09-21T09:51:30.768 INFO:tasks.rados.rados.0.smithi033.stdout:29: writing smithi03385152-29 from 0 to 475136 tid 1 2024-09-21T09:51:30.768 INFO:tasks.rados.rados.0.smithi033.stdout:30: write initial oid 30 2024-09-21T09:51:30.768 INFO:tasks.rados.rados.0.smithi033.stdout:30: seq_num 29 ranges {0=491520} 2024-09-21T09:51:30.771 INFO:tasks.rados.rados.0.smithi033.stdout:30: writing smithi03385152-30 from 0 to 491520 tid 1 2024-09-21T09:51:30.771 INFO:tasks.rados.rados.0.smithi033.stdout:31: write initial oid 31 2024-09-21T09:51:30.771 INFO:tasks.rados.rados.0.smithi033.stdout:31: seq_num 30 ranges {0=507904} 2024-09-21T09:51:30.774 INFO:tasks.rados.rados.0.smithi033.stdout:31: writing smithi03385152-31 from 0 to 507904 tid 1 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:24: finishing write tid 3 to smithi03385152-24 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 6 (ObjNum 23 snap 0 seq_num 23) dirty exists 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:25: finishing write tid 1 to smithi03385152-25 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:25: finishing write tid 2 to smithi03385152-25 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:25: finishing write tid 3 to smithi03385152-25 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 4 (ObjNum 24 snap 0 seq_num 24) dirty exists 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:26: finishing write tid 1 to smithi03385152-26 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:26: finishing write tid 2 to smithi03385152-26 2024-09-21T09:51:30.775 INFO:tasks.rados.rados.0.smithi033.stdout:26: finishing write tid 3 to smithi03385152-26 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 6 (ObjNum 25 snap 0 seq_num 25) dirty exists 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:27: finishing write tid 1 to smithi03385152-27 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:27: finishing write tid 2 to smithi03385152-27 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:27: finishing write tid 3 to smithi03385152-27 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 4 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:28: finishing write tid 1 to smithi03385152-28 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:28: finishing write tid 2 to smithi03385152-28 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:28: finishing write tid 3 to smithi03385152-28 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 2 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:29: finishing write tid 1 to smithi03385152-29 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:29: finishing write tid 2 to smithi03385152-29 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:29: finishing write tid 3 to smithi03385152-29 2024-09-21T09:51:30.776 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 8 (ObjNum 28 snap 0 seq_num 28) dirty exists 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:30: finishing write tid 1 to smithi03385152-30 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:30: finishing write tid 2 to smithi03385152-30 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:24: done (7 left) 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:25: done (6 left) 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:26: done (5 left) 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:27: done (4 left) 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:28: done (3 left) 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:29: done (2 left) 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:32: write initial oid 32 2024-09-21T09:51:30.777 INFO:tasks.rados.rados.0.smithi033.stdout:32: seq_num 31 ranges {0=524288} 2024-09-21T09:51:30.778 INFO:tasks.rados.rados.0.smithi033.stdout:32: writing smithi03385152-32 from 0 to 524288 tid 1 2024-09-21T09:51:30.778 INFO:tasks.rados.rados.0.smithi033.stdout:33: write initial oid 33 2024-09-21T09:51:30.778 INFO:tasks.rados.rados.0.smithi033.stdout:33: seq_num 32 ranges {0=540672} 2024-09-21T09:51:30.782 INFO:tasks.rados.rados.0.smithi033.stdout:33: writing smithi03385152-33 from 0 to 540672 tid 1 2024-09-21T09:51:30.782 INFO:tasks.rados.rados.0.smithi033.stdout:34: write initial oid 34 2024-09-21T09:51:30.782 INFO:tasks.rados.rados.0.smithi033.stdout:34: seq_num 33 ranges {0=557056} 2024-09-21T09:51:30.785 INFO:tasks.rados.rados.0.smithi033.stdout:34: writing smithi03385152-34 from 0 to 557056 tid 1 2024-09-21T09:51:30.785 INFO:tasks.rados.rados.0.smithi033.stdout:35: write initial oid 35 2024-09-21T09:51:30.786 INFO:tasks.rados.rados.0.smithi033.stdout:35: seq_num 34 ranges {0=573440} 2024-09-21T09:51:30.789 INFO:tasks.rados.rados.0.smithi033.stdout:35: writing smithi03385152-35 from 0 to 573440 tid 1 2024-09-21T09:51:30.789 INFO:tasks.rados.rados.0.smithi033.stdout:36: write initial oid 36 2024-09-21T09:51:30.789 INFO:tasks.rados.rados.0.smithi033.stdout:36: seq_num 35 ranges {0=589824} 2024-09-21T09:51:30.792 INFO:tasks.rados.rados.0.smithi033.stdout:36: writing smithi03385152-36 from 0 to 589824 tid 1 2024-09-21T09:51:30.793 INFO:tasks.rados.rados.0.smithi033.stdout:37: write initial oid 37 2024-09-21T09:51:30.793 INFO:tasks.rados.rados.0.smithi033.stdout:37: seq_num 36 ranges {0=606208} 2024-09-21T09:51:30.796 INFO:tasks.rados.rados.0.smithi033.stdout:37: writing smithi03385152-37 from 0 to 606208 tid 1 2024-09-21T09:51:30.796 INFO:tasks.rados.rados.0.smithi033.stdout:38: write initial oid 38 2024-09-21T09:51:30.796 INFO:tasks.rados.rados.0.smithi033.stdout:38: seq_num 37 ranges {0=622592} 2024-09-21T09:51:30.800 INFO:tasks.rados.rados.0.smithi033.stdout:38: writing smithi03385152-38 from 0 to 622592 tid 1 2024-09-21T09:51:30.800 INFO:tasks.rados.rados.0.smithi033.stdout:39: write initial oid 39 2024-09-21T09:51:30.800 INFO:tasks.rados.rados.0.smithi033.stdout:39: seq_num 38 ranges {0=638976} 2024-09-21T09:51:30.804 INFO:tasks.rados.rados.0.smithi033.stdout:39: writing smithi03385152-39 from 0 to 638976 tid 1 2024-09-21T09:51:30.804 INFO:tasks.rados.rados.0.smithi033.stdout:40: write initial oid 40 2024-09-21T09:51:30.804 INFO:tasks.rados.rados.0.smithi033.stdout:30: finishing write tid 3 to smithi03385152-30 2024-09-21T09:51:30.804 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 10 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:51:30.804 INFO:tasks.rados.rados.0.smithi033.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:51:30.804 INFO:tasks.rados.rados.0.smithi033.stdout:31: finishing write tid 1 to smithi03385152-31 2024-09-21T09:51:30.805 INFO:tasks.rados.rados.0.smithi033.stdout:40: seq_num 39 ranges {0=663552} 2024-09-21T09:51:30.808 INFO:tasks.rados.rados.0.smithi033.stdout:40: writing smithi03385152-40 from 0 to 663552 tid 1 2024-09-21T09:51:30.808 INFO:tasks.rados.rados.0.smithi033.stdout:30: done (10 left) 2024-09-21T09:51:30.808 INFO:tasks.rados.rados.0.smithi033.stdout:41: write initial oid 41 2024-09-21T09:51:30.808 INFO:tasks.rados.rados.0.smithi033.stdout:41: seq_num 40 ranges {0=679936} 2024-09-21T09:51:30.812 INFO:tasks.rados.rados.0.smithi033.stdout:41: writing smithi03385152-41 from 0 to 679936 tid 1 2024-09-21T09:51:30.812 INFO:tasks.rados.rados.0.smithi033.stdout:42: write initial oid 42 2024-09-21T09:51:30.812 INFO:tasks.rados.rados.0.smithi033.stdout:42: seq_num 41 ranges {0=696320} 2024-09-21T09:51:30.817 INFO:tasks.rados.rados.0.smithi033.stdout:42: writing smithi03385152-42 from 0 to 696320 tid 1 2024-09-21T09:51:30.817 INFO:tasks.rados.rados.0.smithi033.stdout:43: write initial oid 43 2024-09-21T09:51:30.817 INFO:tasks.rados.rados.0.smithi033.stdout:43: seq_num 42 ranges {0=712704} 2024-09-21T09:51:30.822 INFO:tasks.rados.rados.0.smithi033.stdout:43: writing smithi03385152-43 from 0 to 712704 tid 1 2024-09-21T09:51:30.822 INFO:tasks.rados.rados.0.smithi033.stdout:44: write initial oid 44 2024-09-21T09:51:30.822 INFO:tasks.rados.rados.0.smithi033.stdout:44: seq_num 43 ranges {0=729088} 2024-09-21T09:51:30.827 INFO:tasks.rados.rados.0.smithi033.stdout:44: writing smithi03385152-44 from 0 to 729088 tid 1 2024-09-21T09:51:30.827 INFO:tasks.rados.rados.0.smithi033.stdout:45: write initial oid 45 2024-09-21T09:51:30.827 INFO:tasks.rados.rados.0.smithi033.stdout:45: seq_num 44 ranges {0=745472} 2024-09-21T09:51:30.831 INFO:tasks.rados.rados.0.smithi033.stdout:45: writing smithi03385152-45 from 0 to 745472 tid 1 2024-09-21T09:51:30.831 INFO:tasks.rados.rados.0.smithi033.stdout:46: write initial oid 46 2024-09-21T09:51:30.832 INFO:tasks.rados.rados.0.smithi033.stdout:46: seq_num 45 ranges {0=761856} 2024-09-21T09:51:30.837 INFO:tasks.rados.rados.0.smithi033.stdout:46: writing smithi03385152-46 from 0 to 761856 tid 1 2024-09-21T09:51:30.837 INFO:tasks.rados.rados.0.smithi033.stdout: waiting on 16 2024-09-21T09:51:30.837 INFO:tasks.rados.rados.0.smithi033.stdout:31: finishing write tid 2 to smithi03385152-31 2024-09-21T09:51:30.837 INFO:tasks.rados.rados.0.smithi033.stdout:31: finishing write tid 3 to smithi03385152-31 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 12 (ObjNum 30 snap 0 seq_num 30) dirty exists 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:32: finishing write tid 1 to smithi03385152-32 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:32: finishing write tid 2 to smithi03385152-32 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:32: finishing write tid 3 to smithi03385152-32 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 2 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:33: finishing write tid 1 to smithi03385152-33 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:33: finishing write tid 2 to smithi03385152-33 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:33: finishing write tid 3 to smithi03385152-33 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 6 (ObjNum 32 snap 0 seq_num 32) dirty exists 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:34: finishing write tid 1 to smithi03385152-34 2024-09-21T09:51:30.838 INFO:tasks.rados.rados.0.smithi033.stdout:34: finishing write tid 2 to smithi03385152-34 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:34: finishing write tid 3 to smithi03385152-34 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 8 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:31: done (15 left) 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:32: done (14 left) 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:33: done (13 left) 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:34: done (12 left) 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:47: write initial oid 47 2024-09-21T09:51:30.839 INFO:tasks.rados.rados.0.smithi033.stdout:47: seq_num 46 ranges {0=778240} 2024-09-21T09:51:30.841 INFO:tasks.rados.rados.0.smithi033.stdout:47: writing smithi03385152-47 from 0 to 778240 tid 1 2024-09-21T09:51:30.841 INFO:tasks.rados.rados.0.smithi033.stdout:48: write initial oid 48 2024-09-21T09:51:30.841 INFO:tasks.rados.rados.0.smithi033.stdout:48: seq_num 47 ranges {0=794624} 2024-09-21T09:51:30.846 INFO:tasks.rados.rados.0.smithi033.stdout:48: writing smithi03385152-48 from 0 to 794624 tid 1 2024-09-21T09:51:30.846 INFO:tasks.rados.rados.0.smithi033.stdout:49: write initial oid 49 2024-09-21T09:51:30.846 INFO:tasks.rados.rados.0.smithi033.stdout:49: seq_num 48 ranges {0=409600,409600=401408} 2024-09-21T09:51:30.849 INFO:tasks.rados.rados.0.smithi033.stdout:49: writing smithi03385152-49 from 0 to 409600 tid 1 2024-09-21T09:51:30.851 INFO:tasks.rados.rados.0.smithi033.stdout:49: writing smithi03385152-49 from 409600 to 811008 tid 2 2024-09-21T09:51:30.852 INFO:tasks.rados.rados.0.smithi033.stdout:50: write initial oid 50 2024-09-21T09:51:30.852 INFO:tasks.rados.rados.0.smithi033.stdout:50: seq_num 49 ranges {0=425984,425984=401408} 2024-09-21T09:51:30.854 INFO:tasks.rados.rados.0.smithi033.stdout:50: writing smithi03385152-50 from 0 to 425984 tid 1 2024-09-21T09:51:30.857 INFO:tasks.rados.rados.0.smithi033.stdout:50: writing smithi03385152-50 from 425984 to 827392 tid 2 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout: waiting on 16 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:35: finishing write tid 1 to smithi03385152-35 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:35: finishing write tid 2 to smithi03385152-35 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:35: finishing write tid 3 to smithi03385152-35 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 14 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:36: finishing write tid 1 to smithi03385152-36 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:36: finishing write tid 2 to smithi03385152-36 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:36: finishing write tid 3 to smithi03385152-36 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 16 (ObjNum 35 snap 0 seq_num 35) dirty exists 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:35: done (15 left) 2024-09-21T09:51:30.858 INFO:tasks.rados.rados.0.smithi033.stdout:36: done (14 left) 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:51: copy_from oid 8 from oid 18 current snap is 0 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:37: finishing write tid 1 to smithi03385152-37 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:37: finishing write tid 2 to smithi03385152-37 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:37: finishing write tid 3 to smithi03385152-37 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 4 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:38: finishing write tid 1 to smithi03385152-38 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:37: done (14 left) 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 0 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:38: finishing write tid 2 to smithi03385152-38 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:38: finishing write tid 3 to smithi03385152-38 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 4 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:39: finishing write tid 1 to smithi03385152-39 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:39: finishing write tid 2 to smithi03385152-39 2024-09-21T09:51:30.859 INFO:tasks.rados.rados.0.smithi033.stdout:39: finishing write tid 3 to smithi03385152-39 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 4 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:40: finishing write tid 1 to smithi03385152-40 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:40: finishing write tid 2 to smithi03385152-40 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:40: finishing write tid 3 to smithi03385152-40 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 6 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:41: finishing write tid 1 to smithi03385152-41 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:41: finishing write tid 2 to smithi03385152-41 2024-09-21T09:51:30.860 INFO:tasks.rados.rados.0.smithi033.stdout:52: seq_num 50 ranges {270336=442368,712704=401408} 2024-09-21T09:51:30.861 INFO:tasks.rados.rados.0.smithi033.stdout:52: writing smithi03385152-17 from 270336 to 712704 tid 1 2024-09-21T09:51:30.864 INFO:tasks.rados.rados.0.smithi033.stdout:52: writing smithi03385152-17 from 712704 to 1114112 tid 2 2024-09-21T09:51:30.864 INFO:tasks.rados.rados.0.smithi033.stdout:38: done (14 left) 2024-09-21T09:51:30.864 INFO:tasks.rados.rados.0.smithi033.stdout:39: done (13 left) 2024-09-21T09:51:30.864 INFO:tasks.rados.rados.0.smithi033.stdout:40: done (12 left) 2024-09-21T09:51:30.864 INFO:tasks.rados.rados.0.smithi033.stdout:53: rmattr oid 38 current snap is 0 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:53: done (12 left) 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:54: delete oid 4 current snap is 0 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:41: finishing write tid 3 to smithi03385152-41 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 8 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:42: finishing write tid 1 to smithi03385152-42 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:42: finishing write tid 2 to smithi03385152-42 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:42: finishing write tid 3 to smithi03385152-42 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 10 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:43: finishing write tid 1 to smithi03385152-43 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:43: finishing write tid 2 to smithi03385152-43 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:43: finishing write tid 3 to smithi03385152-43 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 6 (ObjNum 42 snap 0 seq_num 42) dirty exists 2024-09-21T09:51:30.865 INFO:tasks.rados.rados.0.smithi033.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:44: finishing write tid 1 to smithi03385152-44 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:44: finishing write tid 2 to smithi03385152-44 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:44: finishing write tid 3 to smithi03385152-44 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 4 (ObjNum 43 snap 0 seq_num 43) dirty exists 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:45: finishing write tid 1 to smithi03385152-45 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:45: finishing write tid 2 to smithi03385152-45 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:45: finishing write tid 3 to smithi03385152-45 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 10 (ObjNum 44 snap 0 seq_num 44) dirty exists 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:46: finishing write tid 1 to smithi03385152-46 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:46: finishing write tid 2 to smithi03385152-46 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:46: finishing write tid 3 to smithi03385152-46 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 12 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:51:30.866 INFO:tasks.rados.rados.0.smithi033.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:47: finishing write tid 1 to smithi03385152-47 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:47: finishing write tid 2 to smithi03385152-47 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:47: finishing write tid 3 to smithi03385152-47 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 2 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:48: finishing write tid 1 to smithi03385152-48 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:48: finishing write tid 2 to smithi03385152-48 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:49: finishing write tid 1 to smithi03385152-49 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:48: finishing write tid 3 to smithi03385152-48 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 14 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:49: finishing write tid 2 to smithi03385152-49 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:49: finishing write tid 3 to smithi03385152-49 2024-09-21T09:51:30.867 INFO:tasks.rados.rados.0.smithi033.stdout:49: finishing write tid 4 to smithi03385152-49 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 9 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:50: finishing write tid 1 to smithi03385152-50 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:50: finishing write tid 2 to smithi03385152-50 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:50: finishing write tid 3 to smithi03385152-50 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:50: finishing write tid 4 to smithi03385152-50 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 7 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:51: finishing copy_from racing read to smithi03385152-8 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:51: finishing copy_from to smithi03385152-8 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 2 (ObjNum 17 snap 0 seq_num 17) dirty exists 2024-09-21T09:51:30.868 INFO:tasks.rados.rados.0.smithi033.stdout:52: finishing write tid 1 to smithi03385152-17 2024-09-21T09:51:30.869 INFO:tasks.rados.rados.0.smithi033.stdout:41: done (12 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:42: done (11 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:43: done (10 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:44: done (9 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:45: done (8 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:46: done (7 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:47: done (6 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:48: done (5 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:49: done (4 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:50: done (3 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:51: done (2 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:54: done (1 left) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:55: read oid 15 snap -1 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:51:30.870 INFO:tasks.rados.rados.0.smithi033.stdout:56: rollback oid 31 current snap is 0 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:56: done (2 left) 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:57: read oid 28 snap -1 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:58: copy_from oid 29 from oid 47 current snap is 0 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:59: rollback oid 45 current snap is 0 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:59: done (4 left) 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:60: copy_from oid 23 from oid 48 current snap is 0 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:61: delete oid 1 current snap is 0 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:52: finishing write tid 2 to smithi03385152-17 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:52: finishing write tid 3 to smithi03385152-17 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:52: finishing write tid 4 to smithi03385152-17 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 12 (ObjNum 50 snap 0 seq_num 50) dirty exists 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:52: left oid 17 (ObjNum 50 snap 0 seq_num 50) 2024-09-21T09:51:30.871 INFO:tasks.rados.rados.0.smithi033.stdout:57: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:51:30.884 INFO:tasks.rados.rados.0.smithi033.stdout:52: done (5 left) 2024-09-21T09:51:30.884 INFO:tasks.rados.rados.0.smithi033.stdout:57: done (4 left) 2024-09-21T09:51:30.884 INFO:tasks.rados.rados.0.smithi033.stdout:61: done (3 left) 2024-09-21T09:51:30.884 INFO:tasks.rados.rados.0.smithi033.stdout:62: read oid 12 snap -1 2024-09-21T09:51:30.885 INFO:tasks.rados.rados.0.smithi033.stdout:62: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-21T09:51:30.885 INFO:tasks.rados.rados.0.smithi033.stdout:63: snap_create 2024-09-21T09:51:30.885 INFO:tasks.rados.rados.0.smithi033.stdout:55: expect (ObjNum 14 snap 0 seq_num 14) 2024-09-21T09:51:30.891 INFO:tasks.rados.rados.0.smithi033.stdout:58: finishing copy_from racing read to smithi03385152-29 2024-09-21T09:51:30.891 INFO:tasks.rados.rados.0.smithi033.stdout:60: finishing copy_from racing read to smithi03385152-23 2024-09-21T09:51:30.891 INFO:tasks.rados.rados.0.smithi033.stdout:60: finishing copy_from to smithi03385152-23 2024-09-21T09:51:30.892 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 14 (ObjNum 47 snap 0 seq_num 47) dirty exists 2024-09-21T09:51:30.892 INFO:tasks.rados.rados.0.smithi033.stdout:58: finishing copy_from to smithi03385152-29 2024-09-21T09:51:30.892 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:51:30.892 INFO:tasks.rados.rados.0.smithi033.stdout:62: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[27969]: from='client.24769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[27969]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[27969]: pgmap v106: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[27969]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[27969]: Cluster is now healthy 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/79748976' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[39045]: from='client.24769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[39045]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[39045]: pgmap v106: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[39045]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:51:30.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[39045]: Cluster is now healthy 2024-09-21T09:51:30.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:30 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/79748976' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:51:31.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:30 smithi105 ceph-mon[31286]: from='client.24769 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:31.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:30 smithi105 ceph-mon[31286]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:31.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:30 smithi105 ceph-mon[31286]: pgmap v106: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:31.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:30 smithi105 ceph-mon[31286]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T09:51:31.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:30 smithi105 ceph-mon[31286]: Cluster is now healthy 2024-09-21T09:51:31.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:30 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/79748976' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:55: done (4 left) 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:58: done (3 left) 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:60: done (2 left) 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:62: done (1 left) 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:63: done (0 left) 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:64: copy_from oid 15 from oid 2 current snap is 1 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:65: read oid 30 snap -1 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 1 2024-09-21T09:51:31.723 INFO:tasks.rados.rados.0.smithi033.stdout:66: seq_num 53 ranges {794624=491520,1286144=401408} 2024-09-21T09:51:31.730 INFO:tasks.rados.rados.0.smithi033.stdout:66: writing smithi03385152-23 from 794624 to 1286144 tid 1 2024-09-21T09:51:31.734 INFO:tasks.rados.rados.0.smithi033.stdout:66: writing smithi03385152-23 from 1286144 to 1687552 tid 2 2024-09-21T09:51:31.734 INFO:tasks.rados.rados.0.smithi033.stdout:67: rollback oid 10 current snap is 1 2024-09-21T09:51:31.734 INFO:tasks.rados.rados.0.smithi033.stdout:64: finishing copy_from racing read to smithi03385152-15 2024-09-21T09:51:31.734 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 0 2024-09-21T09:51:31.734 INFO:tasks.rados.rados.0.smithi033.stdout:68: setattr oid 32 current snap is 1 2024-09-21T09:51:31.734 INFO:tasks.rados.rados.0.smithi033.stdout:64: finishing copy_from to smithi03385152-15 2024-09-21T09:51:31.734 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:51:31.734 INFO:tasks.rados.rados.0.smithi033.stdout:65: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:51:31.751 INFO:tasks.rados.rados.0.smithi033.stdout:66: finishing write tid 1 to smithi03385152-23 2024-09-21T09:51:31.751 INFO:tasks.rados.rados.0.smithi033.stdout:67: finishing rollback tid 0 to smithi03385152-10 2024-09-21T09:51:31.751 INFO:tasks.rados.rados.0.smithi033.stdout:67: finishing rollback tid 1 to smithi03385152-10 2024-09-21T09:51:31.751 INFO:tasks.rados.rados.0.smithi033.stdout:66: finishing write tid 2 to smithi03385152-23 2024-09-21T09:51:31.751 INFO:tasks.rados.rados.0.smithi033.stdout:66: finishing write tid 3 to smithi03385152-23 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:67: finishing rollback tid 2 to smithi03385152-10 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 10 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:66: finishing write tid 4 to smithi03385152-23 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 17 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:66: left oid 23 (ObjNum 53 snap 1 seq_num 53) 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:64: done (4 left) 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:65: done (3 left) 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:66: done (2 left) 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:67: done (1 left) 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:69: copy_from oid 45 from oid 34 current snap is 1 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:70: read oid 23 snap 0 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:70: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:71: copy_from oid 36 from oid 27 current snap is 1 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:72: rollback oid 49 current snap is 1 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 49 to 0 2024-09-21T09:51:31.752 INFO:tasks.rados.rados.0.smithi033.stdout:73: read oid 4 snap 0 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:73: expect deleted 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:74: rmattr oid 7 current snap is 1 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:74: done (6 left) 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:75: copy_from oid 20 from oid 12 current snap is 1 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:76: read oid 39 snap -1 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:77: copy_from oid 6 from oid 37 current snap is 1 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:78: read oid 2 snap -1 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:78: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:79: setattr oid 41 current snap is 1 2024-09-21T09:51:31.753 INFO:tasks.rados.rados.0.smithi033.stdout:80: snap_create 2024-09-21T09:51:31.754 INFO:tasks.rados.rados.0.smithi033.stdout:72: finishing rollback tid 0 to smithi03385152-49 2024-09-21T09:51:31.757 INFO:tasks.rados.rados.0.smithi033.stdout:72: finishing rollback tid 1 to smithi03385152-49 2024-09-21T09:51:31.757 INFO:tasks.rados.rados.0.smithi033.stdout:72: finishing rollback tid 2 to smithi03385152-49 2024-09-21T09:51:31.757 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 16 (ObjNum 48 snap 0 seq_num 48) dirty exists 2024-09-21T09:51:31.757 INFO:tasks.rados.rados.0.smithi033.stdout:78: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-21T09:51:31.758 INFO:tasks.rados.rados.0.smithi033.stdout:76: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:51:31.779 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 4 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-21T09:51:31.780 INFO:tasks.rados.rados.0.smithi033.stdout:69: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:51:31.780 INFO:tasks.rados.rados.0.smithi033.stdout:70: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:51:31.808 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 12 (ObjNum 40 snap 0 seq_num 40) dirty exists 2024-09-21T09:51:31.808 INFO:tasks.rados.rados.0.smithi033.stdout:69: finishing copy_from to smithi03385152-45 2024-09-21T09:51:31.808 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 8 (ObjNum 33 snap 0 seq_num 33) dirty exists 2024-09-21T09:51:31.808 INFO:tasks.rados.rados.0.smithi033.stdout:71: finishing copy_from racing read to smithi03385152-36 2024-09-21T09:51:31.809 INFO:tasks.rados.rados.0.smithi033.stdout:75: finishing copy_from racing read to smithi03385152-20 2024-09-21T09:51:31.809 INFO:tasks.rados.rados.0.smithi033.stdout:75: finishing copy_from to smithi03385152-20 2024-09-21T09:51:31.809 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 2 (ObjNum 11 snap 0 seq_num 11) dirty exists 2024-09-21T09:51:31.809 INFO:tasks.rados.rados.0.smithi033.stdout:71: finishing copy_from to smithi03385152-36 2024-09-21T09:51:31.809 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 4 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:51:31.809 INFO:tasks.rados.rados.0.smithi033.stdout:77: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:51:31.809 INFO:tasks.rados.rados.0.smithi033.stdout:77: finishing copy_from to smithi03385152-6 2024-09-21T09:51:31.809 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 4 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:51:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:31 smithi033 ceph-mon[39045]: from='client.24872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:31 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/79748976' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:51:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:31 smithi033 ceph-mon[39045]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:51:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:31 smithi033 ceph-mon[27969]: from='client.24872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:31 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/79748976' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:51:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:31 smithi033 ceph-mon[27969]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:51:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:31 smithi105 ceph-mon[31286]: from='client.24872 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:51:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:31 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/79748976' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2024-09-21T09:51:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:31 smithi105 ceph-mon[31286]: osdmap e74: 8 total, 8 up, 8 in 2024-09-21T09:51:32.720 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:51:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:51:32.727 INFO:tasks.rados.rados.0.smithi033.stdout:68: done (11 left) 2024-09-21T09:51:32.727 INFO:tasks.rados.rados.0.smithi033.stdout:69: done (10 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:70: done (9 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:71: done (8 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:72: done (7 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:73: done (6 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:75: done (5 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:76: done (4 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:77: done (3 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:78: done (2 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:79: done (1 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:80: done (0 left) 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:81: rollback oid 40 current snap is 2 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 40 to 1 2024-09-21T09:51:32.728 INFO:tasks.rados.rados.0.smithi033.stdout:82: snap_create 2024-09-21T09:51:32.730 INFO:tasks.rados.rados.0.smithi033.stdout:81: finishing rollback tid 0 to smithi03385152-40 2024-09-21T09:51:32.730 INFO:tasks.rados.rados.0.smithi033.stdout:81: finishing rollback tid 1 to smithi03385152-40 2024-09-21T09:51:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:81: finishing rollback tid 2 to smithi03385152-40 2024-09-21T09:51:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 16 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:51:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:32 smithi033 ceph-mon[27969]: pgmap v108: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:32 smithi033 ceph-mon[27969]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:51:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:32 smithi033 ceph-mon[39045]: pgmap v108: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:32 smithi033 ceph-mon[39045]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:51:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:32 smithi105 ceph-mon[31286]: pgmap v108: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:32 smithi105 ceph-mon[31286]: osdmap e75: 8 total, 8 up, 8 in 2024-09-21T09:51:33.735 INFO:tasks.rados.rados.0.smithi033.stdout:81: done (1 left) 2024-09-21T09:51:33.735 INFO:tasks.rados.rados.0.smithi033.stdout:82: done (0 left) 2024-09-21T09:51:33.736 INFO:tasks.rados.rados.0.smithi033.stdout:83: rollback oid 9 current snap is 3 2024-09-21T09:51:33.736 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 9 to 0 2024-09-21T09:51:33.736 INFO:tasks.rados.rados.0.smithi033.stdout:84: read oid 20 snap 1 2024-09-21T09:51:33.736 INFO:tasks.rados.rados.0.smithi033.stdout:84: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-21T09:51:33.736 INFO:tasks.rados.rados.0.smithi033.stdout:85: read oid 41 snap -1 2024-09-21T09:51:33.736 INFO:tasks.rados.rados.0.smithi033.stdout:85: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:51:33.736 INFO:tasks.rados.rados.0.smithi033.stdout:86: snap_create 2024-09-21T09:51:33.739 INFO:tasks.rados.rados.0.smithi033.stdout:84: expect (ObjNum 11 snap 0 seq_num 11) 2024-09-21T09:51:33.746 INFO:tasks.rados.rados.0.smithi033.stdout:83: finishing rollback tid 0 to smithi03385152-9 2024-09-21T09:51:33.746 INFO:tasks.rados.rados.0.smithi033.stdout:85: expect (ObjNum 40 snap 0 seq_num 40) 2024-09-21T09:51:33.768 INFO:tasks.rados.rados.0.smithi033.stdout:83: finishing rollback tid 1 to smithi03385152-9 2024-09-21T09:51:33.769 INFO:tasks.rados.rados.0.smithi033.stdout:83: finishing rollback tid 2 to smithi03385152-9 2024-09-21T09:51:33.769 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 20 (ObjNum 8 snap 0 seq_num 8) dirty exists 2024-09-21T09:51:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:33 smithi033 ceph-mon[27969]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:51:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:33 smithi033 ceph-mon[39045]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:51:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:33 smithi105 ceph-mon[31286]: osdmap e76: 8 total, 8 up, 8 in 2024-09-21T09:51:34.741 INFO:tasks.rados.rados.0.smithi033.stdout:83: done (3 left) 2024-09-21T09:51:34.741 INFO:tasks.rados.rados.0.smithi033.stdout:84: done (2 left) 2024-09-21T09:51:34.741 INFO:tasks.rados.rados.0.smithi033.stdout:85: done (1 left) 2024-09-21T09:51:34.742 INFO:tasks.rados.rados.0.smithi033.stdout:86: done (0 left) 2024-09-21T09:51:34.742 INFO:tasks.rados.rados.0.smithi033.stdout:87: snap_remove snap 0 2024-09-21T09:51:34.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:34 smithi033 ceph-mon[27969]: pgmap v111: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:34.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:34 smithi033 ceph-mon[27969]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:51:34.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:34 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:34 smithi033 ceph-mon[39045]: pgmap v111: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:34 smithi033 ceph-mon[39045]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:51:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:34 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:34 smithi105 ceph-mon[31286]: pgmap v111: 121 pgs: 121 active+clean; 580 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:34 smithi105 ceph-mon[31286]: osdmap e77: 8 total, 8 up, 8 in 2024-09-21T09:51:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:34 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:35.750 INFO:tasks.rados.rados.0.smithi033.stdout:87: done (0 left) 2024-09-21T09:51:35.750 INFO:tasks.rados.rados.0.smithi033.stdout:88: read oid 29 snap 3 2024-09-21T09:51:35.750 INFO:tasks.rados.rados.0.smithi033.stdout:88: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:51:35.750 INFO:tasks.rados.rados.0.smithi033.stdout:89: read oid 46 snap -1 2024-09-21T09:51:35.750 INFO:tasks.rados.rados.0.smithi033.stdout:89: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:51:35.751 INFO:tasks.rados.rados.0.smithi033.stdout:90: copy_from oid 36 from oid 2 current snap is 4 2024-09-21T09:51:35.751 INFO:tasks.rados.rados.0.smithi033.stdout:91: rmattr oid 49 current snap is 4 2024-09-21T09:51:35.751 INFO:tasks.rados.rados.0.smithi033.stdout:91: done (3 left) 2024-09-21T09:51:35.751 INFO:tasks.rados.rados.0.smithi033.stdout:92: snap_remove snap 2 2024-09-21T09:51:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:35 smithi033 ceph-mon[39045]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:51:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:35 smithi033 ceph-mon[27969]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:51:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:35 smithi105 ceph-mon[31286]: osdmap e78: 8 total, 8 up, 8 in 2024-09-21T09:51:36.752 INFO:tasks.rados.rados.0.smithi033.stdout:92: done (3 left) 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:93: rmattr oid 30 current snap is 4 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:93: done (3 left) 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:94: read oid 42 snap 1 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:94: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:95: rmattr oid 14 current snap is 4 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:95: done (4 left) 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:96: read oid 9 snap 1 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:96: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 4 2024-09-21T09:51:36.753 INFO:tasks.rados.rados.0.smithi033.stdout:97: seq_num 54 ranges {393216=507904,901120=401408} 2024-09-21T09:51:36.758 INFO:tasks.rados.rados.0.smithi033.stdout:97: writing smithi03385152-24 from 393216 to 901120 tid 1 2024-09-21T09:51:36.761 INFO:tasks.rados.rados.0.smithi033.stdout:97: writing smithi03385152-24 from 901120 to 1302528 tid 2 2024-09-21T09:51:36.761 INFO:tasks.rados.rados.0.smithi033.stdout:98: copy_from oid 31 from oid 6 current snap is 4 2024-09-21T09:51:36.761 INFO:tasks.rados.rados.0.smithi033.stdout:89: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:51:36.785 INFO:tasks.rados.rados.0.smithi033.stdout:88: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:51:36.807 INFO:tasks.rados.rados.0.smithi033.stdout:90: finishing copy_from racing read to smithi03385152-36 2024-09-21T09:51:36.807 INFO:tasks.rados.rados.0.smithi033.stdout:90: finishing copy_from to smithi03385152-36 2024-09-21T09:51:36.807 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:51:36.807 INFO:tasks.rados.rados.0.smithi033.stdout:94: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:51:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:96: expect (ObjNum 8 snap 0 seq_num 8) 2024-09-21T09:51:36.830 INFO:tasks.rados.rados.0.smithi033.stdout:88: done (6 left) 2024-09-21T09:51:36.830 INFO:tasks.rados.rados.0.smithi033.stdout:89: done (5 left) 2024-09-21T09:51:36.830 INFO:tasks.rados.rados.0.smithi033.stdout:90: done (4 left) 2024-09-21T09:51:36.830 INFO:tasks.rados.rados.0.smithi033.stdout:94: done (3 left) 2024-09-21T09:51:36.831 INFO:tasks.rados.rados.0.smithi033.stdout:96: done (2 left) 2024-09-21T09:51:36.831 INFO:tasks.rados.rados.0.smithi033.stdout:99: snap_remove snap 3 2024-09-21T09:51:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:36 smithi033 ceph-mon[27969]: pgmap v114: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.3 MiB/s wr, 77 op/s 2024-09-21T09:51:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:36 smithi033 ceph-mon[27969]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:51:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:36 smithi033 ceph-mon[39045]: pgmap v114: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.3 MiB/s wr, 77 op/s 2024-09-21T09:51:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:36 smithi033 ceph-mon[39045]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:51:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:36 smithi105 ceph-mon[31286]: pgmap v114: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.3 MiB/s wr, 77 op/s 2024-09-21T09:51:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:36 smithi105 ceph-mon[31286]: osdmap e79: 8 total, 8 up, 8 in 2024-09-21T09:51:37.765 INFO:tasks.rados.rados.0.smithi033.stdout:99: done (2 left) 2024-09-21T09:51:37.765 INFO:tasks.rados.rados.0.smithi033.stdout:100: copy_from oid 16 from oid 21 current snap is 4 2024-09-21T09:51:37.765 INFO:tasks.rados.rados.0.smithi033.stdout:101: read oid 28 snap 1 2024-09-21T09:51:37.765 INFO:tasks.rados.rados.0.smithi033.stdout:101: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:51:37.765 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 4 2024-09-21T09:51:37.765 INFO:tasks.rados.rados.0.smithi033.stdout:102: seq_num 55 ranges {679936=524288,1204224=401408} 2024-09-21T09:51:37.774 INFO:tasks.rados.rados.0.smithi033.stdout:102: writing smithi03385152-41 from 679936 to 1204224 tid 1 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:102: writing smithi03385152-41 from 1204224 to 1605632 tid 2 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:97: finishing write tid 1 to smithi03385152-24 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:97: finishing write tid 2 to smithi03385152-24 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:97: finishing write tid 3 to smithi03385152-24 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:97: finishing write tid 4 to smithi03385152-24 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 16 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:97: left oid 24 (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:98: finishing copy_from racing read to smithi03385152-31 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:98: finishing copy_from to smithi03385152-31 2024-09-21T09:51:37.777 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 4 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:51:37.778 INFO:tasks.rados.rados.0.smithi033.stdout:101: expect (ObjNum 27 snap 0 seq_num 27) 2024-09-21T09:51:37.791 INFO:tasks.rados.rados.0.smithi033.stdout:100: finishing copy_from racing read to smithi03385152-16 2024-09-21T09:51:37.791 INFO:tasks.rados.rados.0.smithi033.stdout:100: finishing copy_from to smithi03385152-16 2024-09-21T09:51:37.791 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 4 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-21T09:51:37.791 INFO:tasks.rados.rados.0.smithi033.stdout:102: finishing write tid 1 to smithi03385152-41 2024-09-21T09:51:37.791 INFO:tasks.rados.rados.0.smithi033.stdout:102: finishing write tid 2 to smithi03385152-41 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:102: finishing write tid 3 to smithi03385152-41 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:102: finishing write tid 4 to smithi03385152-41 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 32 (ObjNum 55 snap 4 seq_num 55) dirty exists 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:102: left oid 41 (ObjNum 55 snap 4 seq_num 55) 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:97: done (4 left) 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:98: done (3 left) 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:100: done (2 left) 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:101: done (1 left) 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:102: done (0 left) 2024-09-21T09:51:37.792 INFO:tasks.rados.rados.0.smithi033.stdout:103: delete oid 17 current snap is 4 2024-09-21T09:51:37.796 INFO:tasks.rados.rados.0.smithi033.stdout:103: done (0 left) 2024-09-21T09:51:37.796 INFO:tasks.rados.rados.0.smithi033.stdout:104: read oid 43 snap -1 2024-09-21T09:51:37.796 INFO:tasks.rados.rados.0.smithi033.stdout:104: expect (ObjNum 42 snap 0 seq_num 42) 2024-09-21T09:51:37.796 INFO:tasks.rados.rados.0.smithi033.stdout:105: rollback oid 38 current snap is 4 2024-09-21T09:51:37.797 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 1 2024-09-21T09:51:37.797 INFO:tasks.rados.rados.0.smithi033.stdout:106: delete oid 12 current snap is 4 2024-09-21T09:51:37.798 INFO:tasks.rados.rados.0.smithi033.stdout:106: done (2 left) 2024-09-21T09:51:37.798 INFO:tasks.rados.rados.0.smithi033.stdout:107: read oid 17 snap 1 2024-09-21T09:51:37.798 INFO:tasks.rados.rados.0.smithi033.stdout:107: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-21T09:51:37.798 INFO:tasks.rados.rados.0.smithi033.stdout:108: rollback oid 40 current snap is 4 2024-09-21T09:51:37.798 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 40 to 1 2024-09-21T09:51:37.798 INFO:tasks.rados.rados.0.smithi033.stdout:109: delete oid 8 current snap is 4 2024-09-21T09:51:37.799 INFO:tasks.rados.rados.0.smithi033.stdout:104: expect (ObjNum 42 snap 0 seq_num 42) 2024-09-21T09:51:37.821 INFO:tasks.rados.rados.0.smithi033.stdout:105: finishing rollback tid 0 to smithi03385152-38 2024-09-21T09:51:37.821 INFO:tasks.rados.rados.0.smithi033.stdout:105: finishing rollback tid 1 to smithi03385152-38 2024-09-21T09:51:37.821 INFO:tasks.rados.rados.0.smithi033.stdout:104: done (4 left) 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:109: done (3 left) 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:110: setattr oid 46 current snap is 4 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:105: finishing rollback tid 2 to smithi03385152-38 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 36 (ObjNum 37 snap 0 seq_num 37) dirty exists 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:108: finishing rollback tid 0 to smithi03385152-40 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:108: finishing rollback tid 1 to smithi03385152-40 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:108: finishing rollback tid 2 to smithi03385152-40 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 39 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:51:37.822 INFO:tasks.rados.rados.0.smithi033.stdout:107: expect (ObjNum 50 snap 0 seq_num 50) 2024-09-21T09:51:37.857 INFO:tasks.rados.rados.0.smithi033.stdout:105: done (3 left) 2024-09-21T09:51:37.857 INFO:tasks.rados.rados.0.smithi033.stdout:107: done (2 left) 2024-09-21T09:51:37.857 INFO:tasks.rados.rados.0.smithi033.stdout:108: done (1 left) 2024-09-21T09:51:37.857 INFO:tasks.rados.rados.0.smithi033.stdout:111: snap_create 2024-09-21T09:51:37.859 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 20 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:51:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:37 smithi105 ceph-mon[31286]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:51:38.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:37 smithi033 ceph-mon[39045]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:51:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:37 smithi033 ceph-mon[27969]: osdmap e80: 8 total, 8 up, 8 in 2024-09-21T09:51:38.765 INFO:tasks.rados.rados.0.smithi033.stdout:110: done (1 left) 2024-09-21T09:51:38.765 INFO:tasks.rados.rados.0.smithi033.stdout:111: done (0 left) 2024-09-21T09:51:38.765 INFO:tasks.rados.rados.0.smithi033.stdout:112: copy_from oid 25 from oid 6 current snap is 5 2024-09-21T09:51:38.765 INFO:tasks.rados.rados.0.smithi033.stdout:113: setattr oid 23 current snap is 5 2024-09-21T09:51:38.765 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 5 2024-09-21T09:51:38.766 INFO:tasks.rados.rados.0.smithi033.stdout:114: seq_num 59 ranges {24576=598016,622592=401408} 2024-09-21T09:51:38.770 INFO:tasks.rados.rados.0.smithi033.stdout:114: writing smithi03385152-15 from 24576 to 622592 tid 1 2024-09-21T09:51:38.773 INFO:tasks.rados.rados.0.smithi033.stdout:114: writing smithi03385152-15 from 622592 to 1024000 tid 2 2024-09-21T09:51:38.773 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 5 2024-09-21T09:51:38.773 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 19 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:51:38.773 INFO:tasks.rados.rados.0.smithi033.stdout:115: seq_num 60 ranges {188416=614400,802816=401408} 2024-09-21T09:51:38.778 INFO:tasks.rados.rados.0.smithi033.stdout:115: writing smithi03385152-20 from 188416 to 802816 tid 1 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:115: writing smithi03385152-20 from 802816 to 1204224 tid 2 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:113: done (3 left) 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:116: rollback oid 14 current snap is 5 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 14 to 1 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:112: finishing copy_from racing read to smithi03385152-25 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:112: finishing copy_from to smithi03385152-25 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 4 (ObjNum 36 snap 0 seq_num 36) dirty exists 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:114: finishing write tid 1 to smithi03385152-15 2024-09-21T09:51:38.781 INFO:tasks.rados.rados.0.smithi033.stdout:114: finishing write tid 2 to smithi03385152-15 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:114: finishing write tid 3 to smithi03385152-15 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:114: finishing write tid 4 to smithi03385152-15 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 18 (ObjNum 59 snap 5 seq_num 59) dirty exists 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:114: left oid 15 (ObjNum 59 snap 5 seq_num 59) 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:115: finishing write tid 1 to smithi03385152-20 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:112: done (3 left) 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:114: done (2 left) 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:117: read oid 27 snap -1 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:117: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 5 2024-09-21T09:51:38.782 INFO:tasks.rados.rados.0.smithi033.stdout:118: seq_num 61 ranges {344064=630784,974848=401408} 2024-09-21T09:51:38.786 INFO:tasks.rados.rados.0.smithi033.stdout:118: writing smithi03385152-21 from 344064 to 974848 tid 1 2024-09-21T09:51:38.789 INFO:tasks.rados.rados.0.smithi033.stdout:118: writing smithi03385152-21 from 974848 to 1376256 tid 2 2024-09-21T09:51:38.789 INFO:tasks.rados.rados.0.smithi033.stdout:119: snap_create 2024-09-21T09:51:38.789 INFO:tasks.rados.rados.0.smithi033.stdout:115: finishing write tid 2 to smithi03385152-20 2024-09-21T09:51:38.789 INFO:tasks.rados.rados.0.smithi033.stdout:115: finishing write tid 3 to smithi03385152-20 2024-09-21T09:51:38.789 INFO:tasks.rados.rados.0.smithi033.stdout:115: finishing write tid 4 to smithi03385152-20 2024-09-21T09:51:38.790 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 23 (ObjNum 60 snap 5 seq_num 60) dirty exists 2024-09-21T09:51:38.790 INFO:tasks.rados.rados.0.smithi033.stdout:115: left oid 20 (ObjNum 60 snap 5 seq_num 60) 2024-09-21T09:51:38.790 INFO:tasks.rados.rados.0.smithi033.stdout:117: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:51:38.802 INFO:tasks.rados.rados.0.smithi033.stdout:116: finishing rollback tid 0 to smithi03385152-14 2024-09-21T09:51:38.802 INFO:tasks.rados.rados.0.smithi033.stdout:116: finishing rollback tid 1 to smithi03385152-14 2024-09-21T09:51:38.803 INFO:tasks.rados.rados.0.smithi033.stdout:116: finishing rollback tid 2 to smithi03385152-14 2024-09-21T09:51:38.803 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 24 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:51:38.803 INFO:tasks.rados.rados.0.smithi033.stdout:118: finishing write tid 1 to smithi03385152-21 2024-09-21T09:51:38.803 INFO:tasks.rados.rados.0.smithi033.stdout:118: finishing write tid 2 to smithi03385152-21 2024-09-21T09:51:38.803 INFO:tasks.rados.rados.0.smithi033.stdout:118: finishing write tid 3 to smithi03385152-21 2024-09-21T09:51:38.803 INFO:tasks.rados.rados.0.smithi033.stdout:118: finishing write tid 4 to smithi03385152-21 2024-09-21T09:51:38.803 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 27 (ObjNum 61 snap 5 seq_num 61) dirty exists 2024-09-21T09:51:38.803 INFO:tasks.rados.rados.0.smithi033.stdout:118: left oid 21 (ObjNum 61 snap 5 seq_num 61) 2024-09-21T09:51:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:38 smithi105 ceph-mon[31286]: pgmap v117: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.3 MiB/s wr, 77 op/s 2024-09-21T09:51:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:38 smithi105 ceph-mon[31286]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:51:39.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:38 smithi033 ceph-mon[27969]: pgmap v117: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.3 MiB/s wr, 77 op/s 2024-09-21T09:51:39.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:38 smithi033 ceph-mon[27969]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:51:39.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:38 smithi033 ceph-mon[39045]: pgmap v117: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.3 MiB/s wr, 77 op/s 2024-09-21T09:51:39.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:38 smithi033 ceph-mon[39045]: osdmap e81: 8 total, 8 up, 8 in 2024-09-21T09:51:39.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:38 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:38] "GET /metrics HTTP/1.1" 200 238938 "" "Prometheus/2.43.0" 2024-09-21T09:51:39.769 INFO:tasks.rados.rados.0.smithi033.stdout:115: done (4 left) 2024-09-21T09:51:39.769 INFO:tasks.rados.rados.0.smithi033.stdout:116: done (3 left) 2024-09-21T09:51:39.769 INFO:tasks.rados.rados.0.smithi033.stdout:117: done (2 left) 2024-09-21T09:51:39.769 INFO:tasks.rados.rados.0.smithi033.stdout:118: done (1 left) 2024-09-21T09:51:39.769 INFO:tasks.rados.rados.0.smithi033.stdout:119: done (0 left) 2024-09-21T09:51:39.769 INFO:tasks.rados.rados.0.smithi033.stdout:120: read oid 48 snap 4 2024-09-21T09:51:39.769 INFO:tasks.rados.rados.0.smithi033.stdout:120: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:51:39.770 INFO:tasks.rados.rados.0.smithi033.stdout:121: read oid 49 snap 5 2024-09-21T09:51:39.770 INFO:tasks.rados.rados.0.smithi033.stdout:121: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:51:39.770 INFO:tasks.rados.rados.0.smithi033.stdout:122: delete oid 44 current snap is 6 2024-09-21T09:51:39.773 INFO:tasks.rados.rados.0.smithi033.stdout:122: done (2 left) 2024-09-21T09:51:39.773 INFO:tasks.rados.rados.0.smithi033.stdout:123: rollback oid 23 current snap is 6 2024-09-21T09:51:39.773 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 23 to 4 2024-09-21T09:51:39.773 INFO:tasks.rados.rados.0.smithi033.stdout:124: read oid 30 snap -1 2024-09-21T09:51:39.773 INFO:tasks.rados.rados.0.smithi033.stdout:124: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:51:39.773 INFO:tasks.rados.rados.0.smithi033.stdout:125: setattr oid 35 current snap is 6 2024-09-21T09:51:39.774 INFO:tasks.rados.rados.0.smithi033.stdout:126: delete oid 25 current snap is 6 2024-09-21T09:51:39.777 INFO:tasks.rados.rados.0.smithi033.stdout:126: done (5 left) 2024-09-21T09:51:39.777 INFO:tasks.rados.rados.0.smithi033.stdout:127: snap_remove snap 1 2024-09-21T09:51:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:39 smithi105 ceph-mon[31286]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:51:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:39 smithi105 ceph-mon[31286]: pgmap v120: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:39 smithi033 ceph-mon[39045]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:51:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:39 smithi033 ceph-mon[39045]: pgmap v120: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:40.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:39 smithi033 ceph-mon[27969]: osdmap e82: 8 total, 8 up, 8 in 2024-09-21T09:51:40.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:39 smithi033 ceph-mon[27969]: pgmap v120: 121 pgs: 121 active+clean; 25 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:40.779 INFO:tasks.rados.rados.0.smithi033.stdout:127: done (5 left) 2024-09-21T09:51:40.779 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 6 2024-09-21T09:51:40.779 INFO:tasks.rados.rados.0.smithi033.stdout:128: seq_num 64 ranges {40960=679936,720896=401408} 2024-09-21T09:51:40.783 INFO:tasks.rados.rados.0.smithi033.stdout:128: writing smithi03385152-3 from 40960 to 720896 tid 1 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi033.stdout:128: writing smithi03385152-3 from 720896 to 1122304 tid 2 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi033.stdout:123: finishing rollback tid 0 to smithi03385152-23 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi033.stdout:123: finishing rollback tid 1 to smithi03385152-23 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi033.stdout:123: finishing rollback tid 2 to smithi03385152-23 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 31 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:51:40.787 INFO:tasks.rados.rados.0.smithi033.stdout:121: expect (ObjNum 48 snap 0 seq_num 48) 2024-09-21T09:51:40.810 INFO:tasks.rados.rados.0.smithi033.stdout:120: expect (ObjNum 47 snap 0 seq_num 47) 2024-09-21T09:51:40.833 INFO:tasks.rados.rados.0.smithi033.stdout:124: expect (ObjNum 29 snap 0 seq_num 29) 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 35 (ObjNum 34 snap 0 seq_num 34) dirty exists 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:128: finishing write tid 1 to smithi03385152-3 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:128: finishing write tid 2 to smithi03385152-3 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:128: finishing write tid 3 to smithi03385152-3 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:128: finishing write tid 4 to smithi03385152-3 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 32 (ObjNum 64 snap 6 seq_num 64) dirty exists 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:128: left oid 3 (ObjNum 64 snap 6 seq_num 64) 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:120: done (5 left) 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:121: done (4 left) 2024-09-21T09:51:40.847 INFO:tasks.rados.rados.0.smithi033.stdout:123: done (3 left) 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:124: done (2 left) 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:125: done (1 left) 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:128: done (0 left) 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:129: read oid 23 snap 4 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:129: expect (ObjNum 53 snap 1 seq_num 53) 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:130: rollback oid 50 current snap is 6 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 50 to 4 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:131: delete oid 38 current snap is 6 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:131: done (2 left) 2024-09-21T09:51:40.848 INFO:tasks.rados.rados.0.smithi033.stdout:132: delete oid 43 current snap is 6 2024-09-21T09:51:40.849 INFO:tasks.rados.rados.0.smithi033.stdout:130: finishing rollback tid 0 to smithi03385152-50 2024-09-21T09:51:40.852 INFO:tasks.rados.rados.0.smithi033.stdout:132: done (2 left) 2024-09-21T09:51:40.852 INFO:tasks.rados.rados.0.smithi033.stdout:133: copy_from oid 12 from oid 8 current snap is 6 2024-09-21T09:51:40.852 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 6 2024-09-21T09:51:40.852 INFO:tasks.rados.rados.0.smithi033.stdout:134: seq_num 67 ranges {0=729088,729088=401408} 2024-09-21T09:51:40.856 INFO:tasks.rados.rados.0.smithi033.stdout:134: writing smithi03385152-38 from 0 to 729088 tid 1 2024-09-21T09:51:40.859 INFO:tasks.rados.rados.0.smithi033.stdout:134: writing smithi03385152-38 from 729088 to 1130496 tid 2 2024-09-21T09:51:40.859 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 6 2024-09-21T09:51:40.859 INFO:tasks.rados.rados.0.smithi033.stdout:135: seq_num 68 ranges {573440=745472,1318912=401408} 2024-09-21T09:51:40.866 INFO:tasks.rados.rados.0.smithi033.stdout:135: writing smithi03385152-35 from 573440 to 1318912 tid 1 2024-09-21T09:51:40.869 INFO:tasks.rados.rados.0.smithi033.stdout:135: writing smithi03385152-35 from 1318912 to 1720320 tid 2 2024-09-21T09:51:40.869 INFO:tasks.rados.rados.0.smithi033.stdout:136: copy_from oid 31 from oid 19 current snap is 6 2024-09-21T09:51:40.869 INFO:tasks.rados.rados.0.smithi033.stdout:130: finishing rollback tid 1 to smithi03385152-50 2024-09-21T09:51:40.869 INFO:tasks.rados.rados.0.smithi033.stdout:130: finishing rollback tid 2 to smithi03385152-50 2024-09-21T09:51:40.869 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 17 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-21T09:51:40.869 INFO:tasks.rados.rados.0.smithi033.stdout:133: finishing copy_from to smithi03385152-12 2024-09-21T09:51:40.870 INFO:tasks.rados.rados.0.smithi033.stdout:133: got expected ENOENT (src dne) 2024-09-21T09:51:40.870 INFO:tasks.rados.rados.0.smithi033.stdout:133: finishing copy_from racing read to smithi03385152-12 2024-09-21T09:51:40.870 INFO:tasks.rados.rados.0.smithi033.stdout:129: expect (ObjNum 53 snap 1 seq_num 53) 2024-09-21T09:51:40.931 INFO:tasks.rados.rados.0.smithi033.stdout:134: finishing write tid 1 to smithi03385152-38 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:134: finishing write tid 2 to smithi03385152-38 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:134: finishing write tid 3 to smithi03385152-38 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:134: finishing write tid 4 to smithi03385152-38 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 52 (ObjNum 67 snap 6 seq_num 67) dirty exists 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:134: left oid 38 (ObjNum 67 snap 6 seq_num 67) 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:135: finishing write tid 1 to smithi03385152-35 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:129: done (5 left) 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:130: done (4 left) 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:133: done (3 left) 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:134: done (2 left) 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:137: copy_from oid 29 from oid 10 current snap is 6 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:135: finishing write tid 2 to smithi03385152-35 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:135: finishing write tid 3 to smithi03385152-35 2024-09-21T09:51:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:135: finishing write tid 4 to smithi03385152-35 2024-09-21T09:51:40.933 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 46 (ObjNum 68 snap 6 seq_num 68) dirty exists 2024-09-21T09:51:40.933 INFO:tasks.rados.rados.0.smithi033.stdout:135: left oid 35 (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:51:40.933 INFO:tasks.rados.rados.0.smithi033.stdout:136: finishing copy_from racing read to smithi03385152-31 2024-09-21T09:51:40.933 INFO:tasks.rados.rados.0.smithi033.stdout:136: finishing copy_from to smithi03385152-31 2024-09-21T09:51:40.933 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 4 (ObjNum 18 snap 0 seq_num 18) dirty exists 2024-09-21T09:51:40.933 INFO:tasks.rados.rados.0.smithi033.stdout:135: done (2 left) 2024-09-21T09:51:40.933 INFO:tasks.rados.rados.0.smithi033.stdout:136: done (1 left) 2024-09-21T09:51:40.933 INFO:tasks.rados.rados.0.smithi033.stdout:138: snap_create 2024-09-21T09:51:40.934 INFO:tasks.rados.rados.0.smithi033.stdout:137: finishing copy_from racing read to smithi03385152-29 2024-09-21T09:51:40.935 INFO:tasks.rados.rados.0.smithi033.stdout:137: finishing copy_from to smithi03385152-29 2024-09-21T09:51:40.935 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 10 (ObjNum 9 snap 0 seq_num 9) dirty exists 2024-09-21T09:51:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:40 smithi105 ceph-mon[31286]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:51:41.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:40 smithi033 ceph-mon[27969]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:51:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:40 smithi033 ceph-mon[39045]: osdmap e83: 8 total, 8 up, 8 in 2024-09-21T09:51:41.782 INFO:tasks.rados.rados.0.smithi033.stdout:137: done (1 left) 2024-09-21T09:51:41.782 INFO:tasks.rados.rados.0.smithi033.stdout:138: done (0 left) 2024-09-21T09:51:41.782 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 7 2024-09-21T09:51:41.782 INFO:tasks.rados.rados.0.smithi033.stdout:139: seq_num 69 ranges {139264=761856,901120=401408} 2024-09-21T09:51:41.787 INFO:tasks.rados.rados.0.smithi033.stdout:139: writing smithi03385152-9 from 139264 to 901120 tid 1 2024-09-21T09:51:41.790 INFO:tasks.rados.rados.0.smithi033.stdout:139: writing smithi03385152-9 from 901120 to 1302528 tid 2 2024-09-21T09:51:41.790 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 7 2024-09-21T09:51:41.790 INFO:tasks.rados.rados.0.smithi033.stdout:140: seq_num 70 ranges {0=778240,778240=401408} 2024-09-21T09:51:41.796 INFO:tasks.rados.rados.0.smithi033.stdout:140: writing smithi03385152-12 from 0 to 778240 tid 1 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:140: writing smithi03385152-12 from 778240 to 1179648 tid 2 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:141: rollback oid 22 current snap is 7 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 22 to 4 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:139: finishing write tid 1 to smithi03385152-9 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:139: finishing write tid 2 to smithi03385152-9 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:139: finishing write tid 3 to smithi03385152-9 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:139: finishing write tid 4 to smithi03385152-9 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 56 (ObjNum 69 snap 7 seq_num 69) dirty exists 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:139: left oid 9 (ObjNum 69 snap 7 seq_num 69) 2024-09-21T09:51:41.799 INFO:tasks.rados.rados.0.smithi033.stdout:139: done (2 left) 2024-09-21T09:51:41.800 INFO:tasks.rados.rados.0.smithi033.stdout:142: snap_create 2024-09-21T09:51:41.804 INFO:tasks.rados.rados.0.smithi033.stdout:140: finishing write tid 1 to smithi03385152-12 2024-09-21T09:51:41.805 INFO:tasks.rados.rados.0.smithi033.stdout:140: finishing write tid 2 to smithi03385152-12 2024-09-21T09:51:41.805 INFO:tasks.rados.rados.0.smithi033.stdout:140: finishing write tid 3 to smithi03385152-12 2024-09-21T09:51:41.805 INFO:tasks.rados.rados.0.smithi033.stdout:141: finishing rollback tid 0 to smithi03385152-22 2024-09-21T09:51:41.806 INFO:tasks.rados.rados.0.smithi033.stdout:141: finishing rollback tid 1 to smithi03385152-22 2024-09-21T09:51:41.806 INFO:tasks.rados.rados.0.smithi033.stdout:141: finishing rollback tid 2 to smithi03385152-22 2024-09-21T09:51:41.806 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 36 (ObjNum 21 snap 0 seq_num 21) dirty exists 2024-09-21T09:51:41.807 INFO:tasks.rados.rados.0.smithi033.stdout:140: finishing write tid 4 to smithi03385152-12 2024-09-21T09:51:41.807 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 29 (ObjNum 70 snap 7 seq_num 70) dirty exists 2024-09-21T09:51:41.807 INFO:tasks.rados.rados.0.smithi033.stdout:140: left oid 12 (ObjNum 70 snap 7 seq_num 70) 2024-09-21T09:51:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:41 smithi105 ceph-mon[31286]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:51:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:41 smithi105 ceph-mon[31286]: pgmap v123: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 36 op/s 2024-09-21T09:51:42.146 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:41 smithi033 ceph-mon[27969]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:51:42.146 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:41 smithi033 ceph-mon[27969]: pgmap v123: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 36 op/s 2024-09-21T09:51:42.147 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:41 smithi033 ceph-mon[39045]: osdmap e84: 8 total, 8 up, 8 in 2024-09-21T09:51:42.147 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:41 smithi033 ceph-mon[39045]: pgmap v123: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 36 op/s 2024-09-21T09:51:42.778 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:51:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:51:42.791 INFO:tasks.rados.rados.0.smithi033.stdout:140: done (2 left) 2024-09-21T09:51:42.791 INFO:tasks.rados.rados.0.smithi033.stdout:141: done (1 left) 2024-09-21T09:51:42.791 INFO:tasks.rados.rados.0.smithi033.stdout:142: done (0 left) 2024-09-21T09:51:42.791 INFO:tasks.rados.rados.0.smithi033.stdout:143: read oid 33 snap -1 2024-09-21T09:51:42.791 INFO:tasks.rados.rados.0.smithi033.stdout:143: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:51:42.791 INFO:tasks.rados.rados.0.smithi033.stdout:144: read oid 45 snap -1 2024-09-21T09:51:42.791 INFO:tasks.rados.rados.0.smithi033.stdout:144: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:51:42.792 INFO:tasks.rados.rados.0.smithi033.stdout:145: snap_create 2024-09-21T09:51:42.798 INFO:tasks.rados.rados.0.smithi033.stdout:143: expect (ObjNum 32 snap 0 seq_num 32) 2024-09-21T09:51:42.813 INFO:tasks.rados.rados.0.smithi033.stdout:144: expect (ObjNum 33 snap 0 seq_num 33) 2024-09-21T09:51:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:42 smithi105 ceph-mon[31286]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:51:43.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:42 smithi033 ceph-mon[39045]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:51:43.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:42 smithi033 ceph-mon[27969]: osdmap e85: 8 total, 8 up, 8 in 2024-09-21T09:51:43.792 INFO:tasks.rados.rados.0.smithi033.stdout:143: done (2 left) 2024-09-21T09:51:43.793 INFO:tasks.rados.rados.0.smithi033.stdout:144: done (1 left) 2024-09-21T09:51:43.793 INFO:tasks.rados.rados.0.smithi033.stdout:145: done (0 left) 2024-09-21T09:51:43.793 INFO:tasks.rados.rados.0.smithi033.stdout:146: copy_from oid 48 from oid 42 current snap is 9 2024-09-21T09:51:43.793 INFO:tasks.rados.rados.0.smithi033.stdout:147: delete oid 29 current snap is 9 2024-09-21T09:51:43.796 INFO:tasks.rados.rados.0.smithi033.stdout:147: done (1 left) 2024-09-21T09:51:43.796 INFO:tasks.rados.rados.0.smithi033.stdout:148: snap_create 2024-09-21T09:51:43.799 INFO:tasks.rados.rados.0.smithi033.stdout:146: finishing copy_from racing read to smithi03385152-48 2024-09-21T09:51:43.800 INFO:tasks.rados.rados.0.smithi033.stdout:146: finishing copy_from to smithi03385152-48 2024-09-21T09:51:43.800 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 10 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:51:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:43 smithi033 ceph-mon[39045]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:51:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:43 smithi033 ceph-mon[39045]: pgmap v126: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 36 op/s 2024-09-21T09:51:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:43 smithi033 ceph-mon[27969]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:51:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:43 smithi033 ceph-mon[27969]: pgmap v126: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 36 op/s 2024-09-21T09:51:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:43 smithi105 ceph-mon[31286]: osdmap e86: 8 total, 8 up, 8 in 2024-09-21T09:51:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:43 smithi105 ceph-mon[31286]: pgmap v126: 121 pgs: 121 active+clean; 29 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 36 op/s 2024-09-21T09:51:44.797 INFO:tasks.rados.rados.0.smithi033.stdout:146: done (1 left) 2024-09-21T09:51:44.797 INFO:tasks.rados.rados.0.smithi033.stdout:148: done (0 left) 2024-09-21T09:51:44.797 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 10 2024-09-21T09:51:44.797 INFO:tasks.rados.rados.0.smithi033.stdout:149: seq_num 72 ranges {0=409600,409600=524288,933888=278528} 2024-09-21T09:51:44.799 INFO:tasks.rados.rados.0.smithi033.stdout:149: writing smithi03385152-43 from 0 to 409600 tid 1 2024-09-21T09:51:44.802 INFO:tasks.rados.rados.0.smithi033.stdout:149: writing smithi03385152-43 from 409600 to 933888 tid 2 2024-09-21T09:51:44.804 INFO:tasks.rados.rados.0.smithi033.stdout:149: writing smithi03385152-43 from 933888 to 1212416 tid 3 2024-09-21T09:51:44.804 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 10 2024-09-21T09:51:44.804 INFO:tasks.rados.rados.0.smithi033.stdout:149: finishing write tid 1 to smithi03385152-43 2024-09-21T09:51:44.804 INFO:tasks.rados.rados.0.smithi033.stdout:150: seq_num 73 ranges {155648=425984,581632=409600,991232=393216} 2024-09-21T09:51:44.808 INFO:tasks.rados.rados.0.smithi033.stdout:150: writing smithi03385152-10 from 155648 to 581632 tid 1 2024-09-21T09:51:44.810 INFO:tasks.rados.rados.0.smithi033.stdout:150: writing smithi03385152-10 from 581632 to 991232 tid 2 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:150: writing smithi03385152-10 from 991232 to 1384448 tid 3 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:151: snap_create 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:149: finishing write tid 2 to smithi03385152-43 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:149: finishing write tid 3 to smithi03385152-43 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:149: finishing write tid 4 to smithi03385152-43 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:149: finishing write tid 5 to smithi03385152-43 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 12 (ObjNum 72 snap 10 seq_num 72) dirty exists 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:149: left oid 43 (ObjNum 72 snap 10 seq_num 72) 2024-09-21T09:51:44.814 INFO:tasks.rados.rados.0.smithi033.stdout:150: finishing write tid 1 to smithi03385152-10 2024-09-21T09:51:44.815 INFO:tasks.rados.rados.0.smithi033.stdout:150: finishing write tid 2 to smithi03385152-10 2024-09-21T09:51:44.815 INFO:tasks.rados.rados.0.smithi033.stdout:150: finishing write tid 3 to smithi03385152-10 2024-09-21T09:51:44.815 INFO:tasks.rados.rados.0.smithi033.stdout:150: finishing write tid 4 to smithi03385152-10 2024-09-21T09:51:44.817 INFO:tasks.rados.rados.0.smithi033.stdout:150: finishing write tid 5 to smithi03385152-10 2024-09-21T09:51:44.817 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 25 (ObjNum 73 snap 10 seq_num 73) dirty exists 2024-09-21T09:51:44.817 INFO:tasks.rados.rados.0.smithi033.stdout:150: left oid 10 (ObjNum 73 snap 10 seq_num 73) 2024-09-21T09:51:45.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:44 smithi033 ceph-mon[27969]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:51:45.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:44 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:45.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:44 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:45.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:44 smithi033 ceph-mon[39045]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:51:45.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:44 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:45.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:44 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:45.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:44 smithi105 ceph-mon[31286]: osdmap e87: 8 total, 8 up, 8 in 2024-09-21T09:51:45.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:44 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:45.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:44 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:45.806 INFO:tasks.rados.rados.0.smithi033.stdout:149: done (2 left) 2024-09-21T09:51:45.807 INFO:tasks.rados.rados.0.smithi033.stdout:150: done (1 left) 2024-09-21T09:51:45.807 INFO:tasks.rados.rados.0.smithi033.stdout:151: done (0 left) 2024-09-21T09:51:45.807 INFO:tasks.rados.rados.0.smithi033.stdout:152: snap_create 2024-09-21T09:51:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:45 smithi033 ceph-mon[39045]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:51:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:45 smithi033 ceph-mon[39045]: pgmap v129: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:51:46.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:45 smithi033 ceph-mon[27969]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:51:46.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:45 smithi033 ceph-mon[27969]: pgmap v129: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:51:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:45 smithi105 ceph-mon[31286]: osdmap e88: 8 total, 8 up, 8 in 2024-09-21T09:51:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:45 smithi105 ceph-mon[31286]: pgmap v129: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:51:46.461 INFO:tasks.rados.rados.0.smithi033.stdout:152: done (0 left) 2024-09-21T09:51:46.461 INFO:tasks.rados.rados.0.smithi033.stdout:153: read oid 42 snap -1 2024-09-21T09:51:46.461 INFO:tasks.rados.rados.0.smithi033.stdout:153: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:51:46.461 INFO:tasks.rados.rados.0.smithi033.stdout:154: rmattr oid 17 current snap is 12 2024-09-21T09:51:46.462 INFO:tasks.rados.rados.0.smithi033.stdout:154: done (1 left) 2024-09-21T09:51:46.462 INFO:tasks.rados.rados.0.smithi033.stdout:155: setattr oid 28 current snap is 12 2024-09-21T09:51:46.464 INFO:tasks.rados.rados.0.smithi033.stdout:156: copy_from oid 33 from oid 16 current snap is 12 2024-09-21T09:51:46.464 INFO:tasks.rados.rados.0.smithi033.stdout:157: setattr oid 32 current snap is 12 2024-09-21T09:51:46.467 INFO:tasks.rados.rados.0.smithi033.stdout:153: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:51:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 4 (ObjNum 27 snap 0 seq_num 27) dirty exists 2024-09-21T09:51:46.495 INFO:tasks.rados.rados.0.smithi033.stdout:156: finishing copy_from racing read to smithi03385152-33 2024-09-21T09:51:46.495 INFO:tasks.rados.rados.0.smithi033.stdout:156: finishing copy_from to smithi03385152-33 2024-09-21T09:51:46.495 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 4 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-21T09:51:46.495 INFO:tasks.rados.rados.0.smithi033.stdout:153: done (3 left) 2024-09-21T09:51:46.495 INFO:tasks.rados.rados.0.smithi033.stdout:155: done (2 left) 2024-09-21T09:51:46.495 INFO:tasks.rados.rados.0.smithi033.stdout:156: done (1 left) 2024-09-21T09:51:46.495 INFO:tasks.rados.rados.0.smithi033.stdout:158: snap_remove snap 4 2024-09-21T09:51:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:46 smithi033 ceph-mon[39045]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:51:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:46 smithi033 ceph-mon[39045]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:51:47.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:46 smithi033 ceph-mon[27969]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:51:47.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:46 smithi033 ceph-mon[27969]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:51:47.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:46 smithi105 ceph-mon[31286]: osdmap e89: 8 total, 8 up, 8 in 2024-09-21T09:51:47.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:46 smithi105 ceph-mon[31286]: osdmap e90: 8 total, 8 up, 8 in 2024-09-21T09:51:47.469 INFO:tasks.rados.rados.0.smithi033.stdout:158: done (1 left) 2024-09-21T09:51:47.469 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 12 2024-09-21T09:51:47.469 INFO:tasks.rados.rados.0.smithi033.stdout:159: seq_num 74 ranges {557056=442368,999424=696320,1695744=106496} 2024-09-21T09:51:47.476 INFO:tasks.rados.rados.0.smithi033.stdout:159: writing smithi03385152-45 from 557056 to 999424 tid 1 2024-09-21T09:51:47.481 INFO:tasks.rados.rados.0.smithi033.stdout:159: writing smithi03385152-45 from 999424 to 1695744 tid 2 2024-09-21T09:51:47.482 INFO:tasks.rados.rados.0.smithi033.stdout:159: writing smithi03385152-45 from 1695744 to 1802240 tid 3 2024-09-21T09:51:47.482 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 8 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-21T09:51:47.482 INFO:tasks.rados.rados.0.smithi033.stdout:157: done (1 left) 2024-09-21T09:51:47.482 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 12 2024-09-21T09:51:47.482 INFO:tasks.rados.rados.0.smithi033.stdout:160: seq_num 75 ranges {0=458752,458752=581632,1040384=221184} 2024-09-21T09:51:47.485 INFO:tasks.rados.rados.0.smithi033.stdout:160: writing smithi03385152-44 from 0 to 458752 tid 1 2024-09-21T09:51:47.490 INFO:tasks.rados.rados.0.smithi033.stdout:160: writing smithi03385152-44 from 458752 to 1040384 tid 2 2024-09-21T09:51:47.492 INFO:tasks.rados.rados.0.smithi033.stdout:160: writing smithi03385152-44 from 1040384 to 1261568 tid 3 2024-09-21T09:51:47.492 INFO:tasks.rados.rados.0.smithi033.stdout:161: snap_remove snap 11 2024-09-21T09:51:48.475 INFO:tasks.rados.rados.0.smithi033.stdout:161: done (2 left) 2024-09-21T09:51:48.475 INFO:tasks.rados.rados.0.smithi033.stdout:162: delete oid 22 current snap is 12 2024-09-21T09:51:48.475 INFO:tasks.rados.rados.0.smithi033.stdout:159: finishing write tid 1 to smithi03385152-45 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:159: finishing write tid 2 to smithi03385152-45 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:159: finishing write tid 3 to smithi03385152-45 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:159: finishing write tid 4 to smithi03385152-45 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:159: finishing write tid 5 to smithi03385152-45 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 51 (ObjNum 74 snap 12 seq_num 74) dirty exists 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:159: left oid 45 (ObjNum 74 snap 12 seq_num 74) 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:160: finishing write tid 1 to smithi03385152-44 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:160: finishing write tid 2 to smithi03385152-44 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:160: finishing write tid 3 to smithi03385152-44 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:160: finishing write tid 4 to smithi03385152-44 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:160: finishing write tid 5 to smithi03385152-44 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 15 (ObjNum 75 snap 12 seq_num 75) dirty exists 2024-09-21T09:51:48.476 INFO:tasks.rados.rados.0.smithi033.stdout:160: left oid 44 (ObjNum 75 snap 12 seq_num 75) 2024-09-21T09:51:48.478 INFO:tasks.rados.rados.0.smithi033.stdout:159: done (2 left) 2024-09-21T09:51:48.478 INFO:tasks.rados.rados.0.smithi033.stdout:160: done (1 left) 2024-09-21T09:51:48.478 INFO:tasks.rados.rados.0.smithi033.stdout:162: done (0 left) 2024-09-21T09:51:48.479 INFO:tasks.rados.rados.0.smithi033.stdout:163: snap_create 2024-09-21T09:51:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:48 smithi033 ceph-mon[39045]: pgmap v132: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:51:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:48 smithi033 ceph-mon[39045]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:51:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:48 smithi033 ceph-mon[27969]: pgmap v132: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:51:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:48 smithi033 ceph-mon[27969]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:51:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:48 smithi105 ceph-mon[31286]: pgmap v132: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 13 op/s 2024-09-21T09:51:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:48 smithi105 ceph-mon[31286]: osdmap e91: 8 total, 8 up, 8 in 2024-09-21T09:51:49.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:48] "GET /metrics HTTP/1.1" 200 239827 "" "Prometheus/2.43.0" 2024-09-21T09:51:49.479 INFO:tasks.rados.rados.0.smithi033.stdout:163: done (0 left) 2024-09-21T09:51:49.480 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 13 2024-09-21T09:51:49.480 INFO:tasks.rados.rados.0.smithi033.stdout:164: seq_num 77 ranges {303104=491520,794624=794624,1589248=8192} 2024-09-21T09:51:49.484 INFO:tasks.rados.rados.0.smithi033.stdout:164: writing smithi03385152-31 from 303104 to 794624 tid 1 2024-09-21T09:51:49.488 INFO:tasks.rados.rados.0.smithi033.stdout:164: writing smithi03385152-31 from 794624 to 1589248 tid 2 2024-09-21T09:51:49.489 INFO:tasks.rados.rados.0.smithi033.stdout:164: writing smithi03385152-31 from 1589248 to 1597440 tid 3 2024-09-21T09:51:49.489 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 13 2024-09-21T09:51:49.489 INFO:tasks.rados.rados.0.smithi033.stdout:165: seq_num 78 ranges {344064=516096,860160=679936,1540096=122880} 2024-09-21T09:51:49.494 INFO:tasks.rados.rados.0.smithi033.stdout:165: writing smithi03385152-16 from 344064 to 860160 tid 1 2024-09-21T09:51:49.498 INFO:tasks.rados.rados.0.smithi033.stdout:165: writing smithi03385152-16 from 860160 to 1540096 tid 2 2024-09-21T09:51:49.500 INFO:tasks.rados.rados.0.smithi033.stdout:165: writing smithi03385152-16 from 1540096 to 1662976 tid 3 2024-09-21T09:51:49.500 INFO:tasks.rados.rados.0.smithi033.stdout:166: copy_from oid 10 from oid 39 current snap is 13 2024-09-21T09:51:49.500 INFO:tasks.rados.rados.0.smithi033.stdout:164: finishing write tid 1 to smithi03385152-31 2024-09-21T09:51:49.500 INFO:tasks.rados.rados.0.smithi033.stdout:164: finishing write tid 2 to smithi03385152-31 2024-09-21T09:51:49.500 INFO:tasks.rados.rados.0.smithi033.stdout:164: finishing write tid 3 to smithi03385152-31 2024-09-21T09:51:49.500 INFO:tasks.rados.rados.0.smithi033.stdout:164: finishing write tid 4 to smithi03385152-31 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:164: finishing write tid 5 to smithi03385152-31 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 63 (ObjNum 77 snap 13 seq_num 77) dirty exists 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:164: left oid 31 (ObjNum 77 snap 13 seq_num 77) 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:165: finishing write tid 1 to smithi03385152-16 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:164: done (2 left) 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:167: read oid 25 snap 8 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:167: expect deleted 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:168: copy_from oid 21 from oid 35 current snap is 13 2024-09-21T09:51:49.501 INFO:tasks.rados.rados.0.smithi033.stdout:169: snap_remove snap 7 2024-09-21T09:51:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:49 smithi033 ceph-mon[39045]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:51:49.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:49 smithi033 ceph-mon[27969]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:51:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:49 smithi105 ceph-mon[31286]: osdmap e92: 8 total, 8 up, 8 in 2024-09-21T09:51:50.486 INFO:tasks.rados.rados.0.smithi033.stdout:169: done (4 left) 2024-09-21T09:51:50.486 INFO:tasks.rados.rados.0.smithi033.stdout:170: delete oid 8 current snap is 13 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:165: finishing write tid 2 to smithi03385152-16 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:165: finishing write tid 3 to smithi03385152-16 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:165: finishing write tid 4 to smithi03385152-16 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:166: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:166: finishing copy_from to smithi03385152-10 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 4 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:165: finishing write tid 5 to smithi03385152-16 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 68 (ObjNum 78 snap 13 seq_num 78) dirty exists 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:165: left oid 16 (ObjNum 78 snap 13 seq_num 78) 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:168: finishing copy_from racing read to smithi03385152-21 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:168: finishing copy_from to smithi03385152-21 2024-09-21T09:51:50.487 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 46 (ObjNum 68 snap 6 seq_num 68) dirty exists 2024-09-21T09:51:50.490 INFO:tasks.rados.rados.0.smithi033.stdout:165: done (4 left) 2024-09-21T09:51:50.490 INFO:tasks.rados.rados.0.smithi033.stdout:166: done (3 left) 2024-09-21T09:51:50.490 INFO:tasks.rados.rados.0.smithi033.stdout:167: done (2 left) 2024-09-21T09:51:50.490 INFO:tasks.rados.rados.0.smithi033.stdout:168: done (1 left) 2024-09-21T09:51:50.491 INFO:tasks.rados.rados.0.smithi033.stdout:170: done (0 left) 2024-09-21T09:51:50.491 INFO:tasks.rados.rados.0.smithi033.stdout:171: snap_remove snap 8 2024-09-21T09:51:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:50 smithi033 ceph-mon[27969]: pgmap v135: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:50 smithi033 ceph-mon[27969]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:51:50.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:50 smithi033 ceph-mon[39045]: pgmap v135: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:50.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:50 smithi033 ceph-mon[39045]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:51:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:50 smithi105 ceph-mon[31286]: pgmap v135: 121 pgs: 121 active+clean; 35 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:51:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:50 smithi105 ceph-mon[31286]: osdmap e93: 8 total, 8 up, 8 in 2024-09-21T09:51:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:171: done (0 left) 2024-09-21T09:51:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:172: snap_create 2024-09-21T09:51:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:51 smithi033 ceph-mon[39045]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:51:51.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:51 smithi033 ceph-mon[27969]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:51:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:51 smithi105 ceph-mon[31286]: osdmap e94: 8 total, 8 up, 8 in 2024-09-21T09:51:52.494 INFO:tasks.rados.rados.0.smithi033.stdout:172: done (0 left) 2024-09-21T09:51:52.494 INFO:tasks.rados.rados.0.smithi033.stdout:173: read oid 46 snap -1 2024-09-21T09:51:52.494 INFO:tasks.rados.rados.0.smithi033.stdout:173: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:51:52.494 INFO:tasks.rados.rados.0.smithi033.stdout:174: copy_from oid 3 from oid 30 current snap is 14 2024-09-21T09:51:52.494 INFO:tasks.rados.rados.0.smithi033.stdout:175: copy_from oid 17 from oid 9 current snap is 14 2024-09-21T09:51:52.494 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 14 2024-09-21T09:51:52.494 INFO:tasks.rados.rados.0.smithi033.stdout:176: seq_num 80 ranges {1597440=548864,2146304=450560,2596864=352256} 2024-09-21T09:51:52.508 INFO:tasks.rados.rados.0.smithi033.stdout:176: writing smithi03385152-31 from 1597440 to 2146304 tid 1 2024-09-21T09:51:52.511 INFO:tasks.rados.rados.0.smithi033.stdout:176: writing smithi03385152-31 from 2146304 to 2596864 tid 2 2024-09-21T09:51:52.514 INFO:tasks.rados.rados.0.smithi033.stdout:176: writing smithi03385152-31 from 2596864 to 2949120 tid 3 2024-09-21T09:51:52.514 INFO:tasks.rados.rados.0.smithi033.stdout:177: setattr oid 32 current snap is 14 2024-09-21T09:51:52.514 INFO:tasks.rados.rados.0.smithi033.stdout:173: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:51:52.540 INFO:tasks.rados.rados.0.smithi033.stdout:174: finishing copy_from racing read to smithi03385152-3 2024-09-21T09:51:52.540 INFO:tasks.rados.rados.0.smithi033.stdout:174: finishing copy_from to smithi03385152-3 2024-09-21T09:51:52.540 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 10 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:175: finishing copy_from racing read to smithi03385152-17 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:175: finishing copy_from to smithi03385152-17 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 56 (ObjNum 69 snap 7 seq_num 69) dirty exists 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:176: finishing write tid 1 to smithi03385152-31 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:176: finishing write tid 2 to smithi03385152-31 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:176: finishing write tid 3 to smithi03385152-31 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:176: finishing write tid 4 to smithi03385152-31 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:176: finishing write tid 5 to smithi03385152-31 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 85 (ObjNum 80 snap 14 seq_num 80) dirty exists 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:176: left oid 31 (ObjNum 80 snap 14 seq_num 80) 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:173: done (4 left) 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:174: done (3 left) 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:175: done (2 left) 2024-09-21T09:51:52.541 INFO:tasks.rados.rados.0.smithi033.stdout:176: done (1 left) 2024-09-21T09:51:52.542 INFO:tasks.rados.rados.0.smithi033.stdout:178: snap_remove snap 10 2024-09-21T09:51:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:52 smithi033 ceph-mon[39045]: pgmap v138: 121 pgs: 121 active+clean; 42 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 KiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:51:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:52 smithi033 ceph-mon[39045]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:51:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:52 smithi033 ceph-mon[27969]: pgmap v138: 121 pgs: 121 active+clean; 42 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 KiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:51:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:52 smithi033 ceph-mon[27969]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:51:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:52 smithi105 ceph-mon[31286]: pgmap v138: 121 pgs: 121 active+clean; 42 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 KiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:51:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:52 smithi105 ceph-mon[31286]: osdmap e95: 8 total, 8 up, 8 in 2024-09-21T09:51:52.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:51:52 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:52] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:51:53.505 INFO:tasks.rados.rados.0.smithi033.stdout:178: done (1 left) 2024-09-21T09:51:53.506 INFO:tasks.rados.rados.0.smithi033.stdout:179: rmattr oid 36 current snap is 14 2024-09-21T09:51:53.506 INFO:tasks.rados.rados.0.smithi033.stdout:179: done (1 left) 2024-09-21T09:51:53.506 INFO:tasks.rados.rados.0.smithi033.stdout:180: copy_from oid 6 from oid 13 current snap is 14 2024-09-21T09:51:53.506 INFO:tasks.rados.rados.0.smithi033.stdout:181: snap_remove snap 12 2024-09-21T09:51:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:53 smithi105 ceph-mon[31286]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:51:53.793 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:53 smithi033 ceph-mon[39045]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:51:53.793 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:53 smithi033 ceph-mon[27969]: osdmap e96: 8 total, 8 up, 8 in 2024-09-21T09:51:54.507 INFO:tasks.rados.rados.0.smithi033.stdout:181: done (2 left) 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:182: read oid 46 snap -1 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:182: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:183: read oid 35 snap 13 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:183: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:184: copy_from oid 37 from oid 27 current snap is 14 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:185: rollback oid 42 current snap is 14 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 42 to 6 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:186: snap_create 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 18 (ObjNum 31 snap 0 seq_num 31) dirty exists 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:180: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:51:54.508 INFO:tasks.rados.rados.0.smithi033.stdout:180: finishing copy_from to smithi03385152-6 2024-09-21T09:51:54.509 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 4 (ObjNum 12 snap 0 seq_num 12) dirty exists 2024-09-21T09:51:54.516 INFO:tasks.rados.rados.0.smithi033.stdout:182: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:51:54.541 INFO:tasks.rados.rados.0.smithi033.stdout:185: finishing rollback tid 0 to smithi03385152-42 2024-09-21T09:51:54.541 INFO:tasks.rados.rados.0.smithi033.stdout:185: finishing rollback tid 1 to smithi03385152-42 2024-09-21T09:51:54.541 INFO:tasks.rados.rados.0.smithi033.stdout:185: finishing rollback tid 2 to smithi03385152-42 2024-09-21T09:51:54.541 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 64 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:51:54.541 INFO:tasks.rados.rados.0.smithi033.stdout:183: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:51:54.597 INFO:tasks.rados.rados.0.smithi033.stdout:184: finishing copy_from racing read to smithi03385152-37 2024-09-21T09:51:54.597 INFO:tasks.rados.rados.0.smithi033.stdout:184: finishing copy_from to smithi03385152-37 2024-09-21T09:51:54.597 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 4 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:51:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:54 smithi105 ceph-mon[31286]: pgmap v141: 121 pgs: 121 active+clean; 42 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 KiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:51:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:54 smithi105 ceph-mon[31286]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:51:54.973 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:54 smithi033 ceph-mon[39045]: pgmap v141: 121 pgs: 121 active+clean; 42 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 KiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:51:54.973 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:54 smithi033 ceph-mon[39045]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:51:54.973 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:54 smithi033 ceph-mon[27969]: pgmap v141: 121 pgs: 121 active+clean; 42 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 511 KiB/s rd, 1.9 MiB/s wr, 17 op/s 2024-09-21T09:51:54.973 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:54 smithi033 ceph-mon[27969]: osdmap e97: 8 total, 8 up, 8 in 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:177: done (6 left) 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:180: done (5 left) 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:182: done (4 left) 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:183: done (3 left) 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:184: done (2 left) 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:185: done (1 left) 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:186: done (0 left) 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 15 2024-09-21T09:51:55.510 INFO:tasks.rados.rados.0.smithi033.stdout:187: seq_num 81 ranges {204800=565248,770048=737280,1507328=65536} 2024-09-21T09:51:55.514 INFO:tasks.rados.rados.0.smithi033.stdout:187: writing smithi03385152-6 from 204800 to 770048 tid 1 2024-09-21T09:51:55.519 INFO:tasks.rados.rados.0.smithi033.stdout:187: writing smithi03385152-6 from 770048 to 1507328 tid 2 2024-09-21T09:51:55.520 INFO:tasks.rados.rados.0.smithi033.stdout:187: writing smithi03385152-6 from 1507328 to 1572864 tid 3 2024-09-21T09:51:55.520 INFO:tasks.rados.rados.0.smithi033.stdout:188: copy_from oid 30 from oid 4 current snap is 15 2024-09-21T09:51:55.520 INFO:tasks.rados.rados.0.smithi033.stdout:189: snap_create 2024-09-21T09:51:55.522 INFO:tasks.rados.rados.0.smithi033.stdout:187: finishing write tid 1 to smithi03385152-6 2024-09-21T09:51:55.524 INFO:tasks.rados.rados.0.smithi033.stdout:188: finishing copy_from to smithi03385152-30 2024-09-21T09:51:55.524 INFO:tasks.rados.rados.0.smithi033.stdout:188: got expected ENOENT (src dne) 2024-09-21T09:51:55.524 INFO:tasks.rados.rados.0.smithi033.stdout:188: finishing copy_from racing read to smithi03385152-30 2024-09-21T09:51:55.524 INFO:tasks.rados.rados.0.smithi033.stdout:187: finishing write tid 2 to smithi03385152-6 2024-09-21T09:51:55.524 INFO:tasks.rados.rados.0.smithi033.stdout:187: finishing write tid 3 to smithi03385152-6 2024-09-21T09:51:55.524 INFO:tasks.rados.rados.0.smithi033.stdout:187: finishing write tid 4 to smithi03385152-6 2024-09-21T09:51:55.526 INFO:tasks.rados.rados.0.smithi033.stdout:187: finishing write tid 5 to smithi03385152-6 2024-09-21T09:51:55.526 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 31 (ObjNum 81 snap 15 seq_num 81) dirty exists 2024-09-21T09:51:55.526 INFO:tasks.rados.rados.0.smithi033.stdout:187: left oid 6 (ObjNum 81 snap 15 seq_num 81) 2024-09-21T09:51:55.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:55 smithi105 ceph-mon[31286]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:51:55.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:55 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:55.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:55 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:55.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:55 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:55.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:55 smithi105 ceph-mon[31286]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:51:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[39045]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:51:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[39045]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:51:55.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[27969]: osdmap e98: 8 total, 8 up, 8 in 2024-09-21T09:51:55.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:55.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:51:55.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:51:55.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:55 smithi033 ceph-mon[27969]: osdmap e99: 8 total, 8 up, 8 in 2024-09-21T09:51:56.464 INFO:tasks.rados.rados.0.smithi033.stdout:187: done (2 left) 2024-09-21T09:51:56.464 INFO:tasks.rados.rados.0.smithi033.stdout:188: done (1 left) 2024-09-21T09:51:56.464 INFO:tasks.rados.rados.0.smithi033.stdout:189: done (0 left) 2024-09-21T09:51:56.464 INFO:tasks.rados.rados.0.smithi033.stdout:190: delete oid 31 current snap is 16 2024-09-21T09:51:56.468 INFO:tasks.rados.rados.0.smithi033.stdout:190: done (0 left) 2024-09-21T09:51:56.468 INFO:tasks.rados.rados.0.smithi033.stdout:191: delete oid 26 current snap is 16 2024-09-21T09:51:56.472 INFO:tasks.rados.rados.0.smithi033.stdout:191: done (0 left) 2024-09-21T09:51:56.472 INFO:tasks.rados.rados.0.smithi033.stdout:192: copy_from oid 20 from oid 42 current snap is 16 2024-09-21T09:51:56.472 INFO:tasks.rados.rados.0.smithi033.stdout:193: delete oid 5 current snap is 16 2024-09-21T09:51:56.474 INFO:tasks.rados.rados.0.smithi033.stdout:193: done (1 left) 2024-09-21T09:51:56.474 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 16 2024-09-21T09:51:56.474 INFO:tasks.rados.rados.0.smithi033.stdout:194: seq_num 85 ranges {204800=630784,835584=720896,1556480=81920} 2024-09-21T09:51:56.478 INFO:tasks.rados.rados.0.smithi033.stdout:194: writing smithi03385152-13 from 204800 to 835584 tid 1 2024-09-21T09:51:56.483 INFO:tasks.rados.rados.0.smithi033.stdout:194: writing smithi03385152-13 from 835584 to 1556480 tid 2 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:194: writing smithi03385152-13 from 1556480 to 1638400 tid 3 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:195: read oid 19 snap -1 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:195: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 16 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:192: finishing copy_from racing read to smithi03385152-20 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:192: finishing copy_from to smithi03385152-20 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 64 (ObjNum 41 snap 0 seq_num 41) dirty exists 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:194: finishing write tid 1 to smithi03385152-13 2024-09-21T09:51:56.484 INFO:tasks.rados.rados.0.smithi033.stdout:196: seq_num 86 ranges {1720320=647168,2367488=606208,2973696=196608} 2024-09-21T09:51:56.497 INFO:tasks.rados.rados.0.smithi033.stdout:196: writing smithi03385152-21 from 1720320 to 2367488 tid 1 2024-09-21T09:51:56.500 INFO:tasks.rados.rados.0.smithi033.stdout:196: writing smithi03385152-21 from 2367488 to 2973696 tid 2 2024-09-21T09:51:56.502 INFO:tasks.rados.rados.0.smithi033.stdout:196: writing smithi03385152-21 from 2973696 to 3170304 tid 3 2024-09-21T09:51:56.502 INFO:tasks.rados.rados.0.smithi033.stdout:192: done (3 left) 2024-09-21T09:51:56.502 INFO:tasks.rados.rados.0.smithi033.stdout:197: delete oid 11 current snap is 16 2024-09-21T09:51:56.502 INFO:tasks.rados.rados.0.smithi033.stdout:194: finishing write tid 2 to smithi03385152-13 2024-09-21T09:51:56.502 INFO:tasks.rados.rados.0.smithi033.stdout:194: finishing write tid 3 to smithi03385152-13 2024-09-21T09:51:56.502 INFO:tasks.rados.rados.0.smithi033.stdout:195: expect (ObjNum 18 snap 0 seq_num 18) 2024-09-21T09:51:56.511 INFO:tasks.rados.rados.0.smithi033.stdout:194: finishing write tid 4 to smithi03385152-13 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:194: finishing write tid 5 to smithi03385152-13 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 100 (ObjNum 85 snap 16 seq_num 85) dirty exists 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:194: left oid 13 (ObjNum 85 snap 16 seq_num 85) 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:196: finishing write tid 1 to smithi03385152-21 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:194: done (3 left) 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:195: done (2 left) 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:197: done (1 left) 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:198: read oid 32 snap -1 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:198: expect (ObjNum 31 snap 0 seq_num 31) 2024-09-21T09:51:56.512 INFO:tasks.rados.rados.0.smithi033.stdout:199: read oid 1 snap 13 2024-09-21T09:51:56.513 INFO:tasks.rados.rados.0.smithi033.stdout:199: expect deleted 2024-09-21T09:51:56.513 INFO:tasks.rados.rados.0.smithi033.stdout:200: snap_remove snap 6 2024-09-21T09:51:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:56 smithi105 ceph-mon[31286]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:51:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:56 smithi105 ceph-mon[31286]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:51:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:56 smithi105 ceph-mon[31286]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:51:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:56 smithi105 ceph-mon[31286]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi105 2024-09-21T09:51:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:56 smithi105 ceph-mon[31286]: pgmap v144: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 926 KiB/s wr, 11 op/s 2024-09-21T09:51:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:56 smithi105 ceph-mon[31286]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:51:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[39045]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:51:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[39045]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:51:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[39045]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:51:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[39045]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi105 2024-09-21T09:51:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[39045]: pgmap v144: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 926 KiB/s wr, 11 op/s 2024-09-21T09:51:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[39045]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:51:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[27969]: Upgrade: Target is version 19.3.0-5092-g62da54b4 (unknown) 2024-09-21T09:51:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[27969]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, digests ['quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47'] 2024-09-21T09:51:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[27969]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-21T09:51:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[27969]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 on smithi105 2024-09-21T09:51:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[27969]: pgmap v144: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 926 KiB/s wr, 11 op/s 2024-09-21T09:51:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:56 smithi033 ceph-mon[27969]: osdmap e100: 8 total, 8 up, 8 in 2024-09-21T09:51:57.472 INFO:tasks.rados.rados.0.smithi033.stdout:200: done (3 left) 2024-09-21T09:51:57.473 INFO:tasks.rados.rados.0.smithi033.stdout:201: copy_from oid 5 from oid 25 current snap is 16 2024-09-21T09:51:57.473 INFO:tasks.rados.rados.0.smithi033.stdout:196: finishing write tid 2 to smithi03385152-21 2024-09-21T09:51:57.473 INFO:tasks.rados.rados.0.smithi033.stdout:196: finishing write tid 3 to smithi03385152-21 2024-09-21T09:51:57.473 INFO:tasks.rados.rados.0.smithi033.stdout:196: finishing write tid 4 to smithi03385152-21 2024-09-21T09:51:57.473 INFO:tasks.rados.rados.0.smithi033.stdout:196: finishing write tid 5 to smithi03385152-21 2024-09-21T09:51:57.473 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 68 (ObjNum 86 snap 16 seq_num 86) dirty exists 2024-09-21T09:51:57.473 INFO:tasks.rados.rados.0.smithi033.stdout:196: left oid 21 (ObjNum 86 snap 16 seq_num 86) 2024-09-21T09:51:57.473 INFO:tasks.rados.rados.0.smithi033.stdout:198: expect (ObjNum 31 snap 0 seq_num 31) 2024-09-21T09:51:57.494 INFO:tasks.rados.rados.0.smithi033.stdout:201: finishing copy_from to smithi03385152-5 2024-09-21T09:51:57.494 INFO:tasks.rados.rados.0.smithi033.stdout:201: got expected ENOENT (src dne) 2024-09-21T09:51:57.494 INFO:tasks.rados.rados.0.smithi033.stdout:201: finishing copy_from racing read to smithi03385152-5 2024-09-21T09:51:57.494 INFO:tasks.rados.rados.0.smithi033.stdout:196: done (3 left) 2024-09-21T09:51:57.494 INFO:tasks.rados.rados.0.smithi033.stdout:198: done (2 left) 2024-09-21T09:51:57.494 INFO:tasks.rados.rados.0.smithi033.stdout:199: done (1 left) 2024-09-21T09:51:57.495 INFO:tasks.rados.rados.0.smithi033.stdout:201: done (0 left) 2024-09-21T09:51:57.495 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 16 2024-09-21T09:51:57.495 INFO:tasks.rados.rados.0.smithi033.stdout:202: seq_num 88 ranges {557056=679936,1236992=778240,2015232=24576} 2024-09-21T09:51:57.501 INFO:tasks.rados.rados.0.smithi033.stdout:202: writing smithi03385152-34 from 557056 to 1236992 tid 1 2024-09-21T09:51:57.505 INFO:tasks.rados.rados.0.smithi033.stdout:202: writing smithi03385152-34 from 1236992 to 2015232 tid 2 2024-09-21T09:51:57.505 INFO:tasks.rados.rados.0.smithi033.stdout:202: writing smithi03385152-34 from 2015232 to 2039808 tid 3 2024-09-21T09:51:57.505 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 16 2024-09-21T09:51:57.506 INFO:tasks.rados.rados.0.smithi033.stdout:203: seq_num 89 ranges {811008=696320,1507328=655360,2162688=147456} 2024-09-21T09:51:57.513 INFO:tasks.rados.rados.0.smithi033.stdout:203: writing smithi03385152-49 from 811008 to 1507328 tid 1 2024-09-21T09:51:57.517 INFO:tasks.rados.rados.0.smithi033.stdout:203: writing smithi03385152-49 from 1507328 to 2162688 tid 2 2024-09-21T09:51:57.518 INFO:tasks.rados.rados.0.smithi033.stdout:203: writing smithi03385152-49 from 2162688 to 2310144 tid 3 2024-09-21T09:51:57.518 INFO:tasks.rados.rados.0.smithi033.stdout:204: read oid 5 snap -1 2024-09-21T09:51:57.518 INFO:tasks.rados.rados.0.smithi033.stdout:204: expect deleted 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:205: read oid 1 snap 9 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:205: expect deleted 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:206: setattr oid 37 current snap is 16 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:202: finishing write tid 1 to smithi03385152-34 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:202: finishing write tid 2 to smithi03385152-34 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:202: finishing write tid 3 to smithi03385152-34 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:202: finishing write tid 4 to smithi03385152-34 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:202: finishing write tid 5 to smithi03385152-34 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 90 (ObjNum 88 snap 16 seq_num 88) dirty exists 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:202: left oid 34 (ObjNum 88 snap 16 seq_num 88) 2024-09-21T09:51:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:203: finishing write tid 1 to smithi03385152-49 2024-09-21T09:51:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:202: done (4 left) 2024-09-21T09:51:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:204: done (3 left) 2024-09-21T09:51:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:205: done (2 left) 2024-09-21T09:51:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:207: rollback oid 23 current snap is 16 2024-09-21T09:51:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 23 to 5 2024-09-21T09:51:57.521 INFO:tasks.rados.rados.0.smithi033.stdout:208: delete oid 31 current snap is 16 2024-09-21T09:51:57.521 INFO:tasks.rados.rados.0.smithi033.stdout:203: finishing write tid 2 to smithi03385152-49 2024-09-21T09:51:57.521 INFO:tasks.rados.rados.0.smithi033.stdout:203: finishing write tid 3 to smithi03385152-49 2024-09-21T09:51:57.521 INFO:tasks.rados.rados.0.smithi033.stdout:203: finishing write tid 4 to smithi03385152-49 2024-09-21T09:51:57.522 INFO:tasks.rados.rados.0.smithi033.stdout:208: done (3 left) 2024-09-21T09:51:57.522 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 16 2024-09-21T09:51:57.522 INFO:tasks.rados.rados.0.smithi033.stdout:209: seq_num 91 ranges {0=729088,729088=425984,1155072=376832} 2024-09-21T09:51:57.526 INFO:tasks.rados.rados.0.smithi033.stdout:209: writing smithi03385152-4 from 0 to 729088 tid 1 2024-09-21T09:51:57.528 INFO:tasks.rados.rados.0.smithi033.stdout:209: writing smithi03385152-4 from 729088 to 1155072 tid 2 2024-09-21T09:51:57.531 INFO:tasks.rados.rados.0.smithi033.stdout:209: writing smithi03385152-4 from 1155072 to 1531904 tid 3 2024-09-21T09:51:57.531 INFO:tasks.rados.rados.0.smithi033.stdout:210: setattr oid 1 current snap is 16 2024-09-21T09:51:57.531 INFO:tasks.rados.rados.0.smithi033.stdout:203: finishing write tid 5 to smithi03385152-49 2024-09-21T09:51:57.531 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 61 (ObjNum 89 snap 16 seq_num 89) dirty exists 2024-09-21T09:51:57.532 INFO:tasks.rados.rados.0.smithi033.stdout:203: left oid 49 (ObjNum 89 snap 16 seq_num 89) 2024-09-21T09:51:57.532 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 24 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:51:57.532 INFO:tasks.rados.rados.0.smithi033.stdout:207: finishing rollback tid 0 to smithi03385152-23 2024-09-21T09:51:57.532 INFO:tasks.rados.rados.0.smithi033.stdout:207: finishing rollback tid 1 to smithi03385152-23 2024-09-21T09:51:57.532 INFO:tasks.rados.rados.0.smithi033.stdout:207: finishing rollback tid 2 to smithi03385152-23 2024-09-21T09:51:57.532 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 71 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:51:57.532 INFO:tasks.rados.rados.0.smithi033.stdout:209: finishing write tid 1 to smithi03385152-4 2024-09-21T09:51:57.532 INFO:tasks.rados.rados.0.smithi033.stdout:209: finishing write tid 2 to smithi03385152-4 2024-09-21T09:51:57.533 INFO:tasks.rados.rados.0.smithi033.stdout:203: done (4 left) 2024-09-21T09:51:57.533 INFO:tasks.rados.rados.0.smithi033.stdout:206: done (3 left) 2024-09-21T09:51:57.533 INFO:tasks.rados.rados.0.smithi033.stdout:207: done (2 left) 2024-09-21T09:51:57.533 INFO:tasks.rados.rados.0.smithi033.stdout:211: setattr oid 30 current snap is 16 2024-09-21T09:51:57.533 INFO:tasks.rados.rados.0.smithi033.stdout:209: finishing write tid 3 to smithi03385152-4 2024-09-21T09:51:57.533 INFO:tasks.rados.rados.0.smithi033.stdout:209: finishing write tid 4 to smithi03385152-4 2024-09-21T09:51:57.534 INFO:tasks.rados.rados.0.smithi033.stdout:212: snap_remove snap 5 2024-09-21T09:51:58.523 INFO:tasks.rados.rados.0.smithi033.stdout:212: done (3 left) 2024-09-21T09:51:58.523 INFO:tasks.rados.rados.0.smithi033.stdout:213: read oid 49 snap -1 2024-09-21T09:51:58.523 INFO:tasks.rados.rados.0.smithi033.stdout:213: expect (ObjNum 89 snap 16 seq_num 89) 2024-09-21T09:51:58.523 INFO:tasks.rados.rados.0.smithi033.stdout:214: read oid 43 snap -1 2024-09-21T09:51:58.523 INFO:tasks.rados.rados.0.smithi033.stdout:214: expect (ObjNum 72 snap 10 seq_num 72) 2024-09-21T09:51:58.523 INFO:tasks.rados.rados.0.smithi033.stdout:215: read oid 33 snap -1 2024-09-21T09:51:58.523 INFO:tasks.rados.rados.0.smithi033.stdout:215: expect (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:51:58.523 INFO:tasks.rados.rados.0.smithi033.stdout:216: snap_remove snap 15 2024-09-21T09:51:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:58 smithi033 ceph-mon[39045]: pgmap v147: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 926 KiB/s wr, 11 op/s 2024-09-21T09:51:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:58 smithi033 ceph-mon[39045]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:51:58.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:58 smithi033 ceph-mon[27969]: pgmap v147: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 926 KiB/s wr, 11 op/s 2024-09-21T09:51:58.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:58 smithi033 ceph-mon[27969]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:51:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:58 smithi105 ceph-mon[31286]: pgmap v147: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 926 KiB/s wr, 11 op/s 2024-09-21T09:51:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:58 smithi105 ceph-mon[31286]: osdmap e101: 8 total, 8 up, 8 in 2024-09-21T09:51:59.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:51:58 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:51:58] "GET /metrics HTTP/1.1" 200 239827 "" "Prometheus/2.43.0" 2024-09-21T09:51:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:216: done (6 left) 2024-09-21T09:51:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:217: copy_from oid 16 from oid 41 current snap is 16 2024-09-21T09:51:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 26 (ObjNum 49 snap 0 seq_num 90809136) dirty exists 2024-09-21T09:51:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 111 (ObjNum 29 snap 0 seq_num 29) dirty exists 2024-09-21T09:51:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:209: finishing write tid 5 to smithi03385152-4 2024-09-21T09:51:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 19 (ObjNum 91 snap 16 seq_num 91) dirty exists 2024-09-21T09:51:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:209: left oid 4 (ObjNum 91 snap 16 seq_num 91) 2024-09-21T09:51:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:213: expect (ObjNum 89 snap 16 seq_num 89) 2024-09-21T09:51:59.607 INFO:tasks.rados.rados.0.smithi033.stdout:215: expect (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:51:59.616 INFO:tasks.rados.rados.0.smithi033.stdout:214: expect (ObjNum 72 snap 10 seq_num 72) 2024-09-21T09:51:59.648 INFO:tasks.rados.rados.0.smithi033.stdout:217: finishing copy_from racing read to smithi03385152-16 2024-09-21T09:51:59.648 INFO:tasks.rados.rados.0.smithi033.stdout:217: finishing copy_from to smithi03385152-16 2024-09-21T09:51:59.648 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 32 (ObjNum 55 snap 4 seq_num 55) dirty exists 2024-09-21T09:51:59.648 INFO:tasks.rados.rados.0.smithi033.stdout:209: done (6 left) 2024-09-21T09:51:59.648 INFO:tasks.rados.rados.0.smithi033.stdout:210: done (5 left) 2024-09-21T09:51:59.648 INFO:tasks.rados.rados.0.smithi033.stdout:211: done (4 left) 2024-09-21T09:51:59.649 INFO:tasks.rados.rados.0.smithi033.stdout:213: done (3 left) 2024-09-21T09:51:59.649 INFO:tasks.rados.rados.0.smithi033.stdout:214: done (2 left) 2024-09-21T09:51:59.649 INFO:tasks.rados.rados.0.smithi033.stdout:215: done (1 left) 2024-09-21T09:51:59.649 INFO:tasks.rados.rados.0.smithi033.stdout:217: done (0 left) 2024-09-21T09:51:59.649 INFO:tasks.rados.rados.0.smithi033.stdout:218: read oid 27 snap -1 2024-09-21T09:51:59.650 INFO:tasks.rados.rados.0.smithi033.stdout:218: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:51:59.650 INFO:tasks.rados.rados.0.smithi033.stdout:219: snap_remove snap 9 2024-09-21T09:51:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:59 smithi105 ceph-mon[31286]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:51:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:51:59 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:59 smithi033 ceph-mon[39045]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:51:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:51:59 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:51:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:59 smithi033 ceph-mon[27969]: osdmap e102: 8 total, 8 up, 8 in 2024-09-21T09:51:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:51:59 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:00.222 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:52:00.538 INFO:tasks.rados.rados.0.smithi033.stdout:219: done (1 left) 2024-09-21T09:52:00.538 INFO:tasks.rados.rados.0.smithi033.stdout:220: read oid 33 snap -1 2024-09-21T09:52:00.538 INFO:tasks.rados.rados.0.smithi033.stdout:220: expect (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:52:00.538 INFO:tasks.rados.rados.0.smithi033.stdout:221: delete oid 36 current snap is 16 2024-09-21T09:52:00.538 INFO:tasks.rados.rados.0.smithi033.stdout:218: expect (ObjNum 26 snap 0 seq_num 26) 2024-09-21T09:52:00.557 INFO:tasks.rados.rados.0.smithi033.stdout:220: expect (ObjNum 20 snap 0 seq_num 20) 2024-09-21T09:52:00.566 INFO:tasks.rados.rados.0.smithi033.stdout:218: done (2 left) 2024-09-21T09:52:00.566 INFO:tasks.rados.rados.0.smithi033.stdout:220: done (1 left) 2024-09-21T09:52:00.566 INFO:tasks.rados.rados.0.smithi033.stdout:221: done (0 left) 2024-09-21T09:52:00.566 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 16 2024-09-21T09:52:00.566 INFO:tasks.rados.rados.0.smithi033.stdout:222: seq_num 93 ranges {491520=761856,1253376=647168,1900544=155648} 2024-09-21T09:52:00.573 INFO:tasks.rados.rados.0.smithi033.stdout:222: writing smithi03385152-30 from 491520 to 1253376 tid 1 2024-09-21T09:52:00.577 INFO:tasks.rados.rados.0.smithi033.stdout:222: writing smithi03385152-30 from 1253376 to 1900544 tid 2 2024-09-21T09:52:00.578 INFO:tasks.rados.rados.0.smithi033.stdout:222: writing smithi03385152-30 from 1900544 to 2056192 tid 3 2024-09-21T09:52:00.578 INFO:tasks.rados.rados.0.smithi033.stdout:223: rmattr oid 17 current snap is 16 2024-09-21T09:52:00.578 INFO:tasks.rados.rados.0.smithi033.stdout:223: done (1 left) 2024-09-21T09:52:00.578 INFO:tasks.rados.rados.0.smithi033.stdout:224: snap_remove snap 13 2024-09-21T09:52:00.578 INFO:tasks.rados.rados.0.smithi033.stdout:222: finishing write tid 1 to smithi03385152-30 2024-09-21T09:52:00.768 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:52:00.768 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (2m) 119s ago 3m 17.3M - 0.25.0 c8568f914cd2 f5c871f25675 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (3m) 2m ago 3m 82.5M - 9.4.7 954c08fa6188 39f7f9958856 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (4m) 119s ago 4m 13.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (4m) 2m ago 4m 16.8M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443 running (8m) 2m ago 8m 399M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be e1080a983fa6 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:9283,8765 running (9m) 119s ago 9m 445M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 21061eedc154 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (10m) 119s ago 10m 48.2M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 220cb0bc67bb 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (8m) 2m ago 8m 40.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c2741af3f853 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (8m) 119s ago 8m 41.4M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f8b68f40cc5 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (2m) 119s ago 2m 13.0M - 1.5.0 0da6a335fe13 ae258c372f47 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (2m) 2m ago 2m 3376k - 1.5.0 0da6a335fe13 446328798f33 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (7m) 119s ago 7m 43.9M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (7m) 119s ago 7m 44.3M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (7m) 119s ago 7m 44.8M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:52:00.769 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (6m) 119s ago 6m 39.8M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:52:00.770 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (6m) 2m ago 6m 42.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:52:00.770 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (6m) 2m ago 6m 42.6M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:52:00.770 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (5m) 2m ago 5m 42.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:52:00.770 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (5m) 2m ago 5m 42.4M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:52:00.770 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (2m) 2m ago 3m 24.0M - 2.43.0 a07b618ecd1d 9b3306609c60 2024-09-21T09:52:00.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:00 smithi105 ceph-mon[31286]: pgmap v150: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:00 smithi105 ceph-mon[31286]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:52:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:00 smithi033 ceph-mon[39045]: pgmap v150: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:00 smithi033 ceph-mon[39045]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:52:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:00 smithi033 ceph-mon[27969]: pgmap v150: 121 pgs: 121 active+clean; 45 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:00 smithi033 ceph-mon[27969]: osdmap e103: 8 total, 8 up, 8 in 2024-09-21T09:52:01.092 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:52:01.092 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:52:01.092 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 15 2024-09-21T09:52:01.093 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:52:01.094 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:52:01.365 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:52:01.365 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:52:01.365 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:52:01.365 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:52:01.365 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-21T09:52:01.365 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "0/20 daemons upgraded", 2024-09-21T09:52:01.365 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Pulling quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 image on host smithi105", 2024-09-21T09:52:01.366 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:52:01.366 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:52:01.548 INFO:tasks.rados.rados.0.smithi033.stdout:224: done (1 left) 2024-09-21T09:52:01.548 INFO:tasks.rados.rados.0.smithi033.stdout:225: snap_remove snap 14 2024-09-21T09:52:01.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:01 smithi033 ceph-mon[39045]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:01.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:01 smithi033 ceph-mon[39045]: from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:01.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:01 smithi033 ceph-mon[39045]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:52:01.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:01 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/4268546125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:01 smithi033 ceph-mon[27969]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:01 smithi033 ceph-mon[27969]: from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:01.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:01 smithi033 ceph-mon[27969]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:52:01.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:01 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/4268546125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:01 smithi105 ceph-mon[31286]: from='client.14982 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:01 smithi105 ceph-mon[31286]: from='client.24890 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:01 smithi105 ceph-mon[31286]: osdmap e104: 8 total, 8 up, 8 in 2024-09-21T09:52:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:01 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/4268546125' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:02.552 INFO:tasks.rados.rados.0.smithi033.stdout:225: done (1 left) 2024-09-21T09:52:02.552 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 16 2024-09-21T09:52:02.552 INFO:tasks.rados.rados.0.smithi033.stdout:222: finishing write tid 2 to smithi03385152-30 2024-09-21T09:52:02.552 INFO:tasks.rados.rados.0.smithi033.stdout:226: seq_num 94 ranges {1572864=778240,2351104=532480,2883584=270336} 2024-09-21T09:52:02.566 INFO:tasks.rados.rados.0.smithi033.stdout:226: writing smithi03385152-6 from 1572864 to 2351104 tid 1 2024-09-21T09:52:02.569 INFO:tasks.rados.rados.0.smithi033.stdout:226: writing smithi03385152-6 from 2351104 to 2883584 tid 2 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:226: writing smithi03385152-6 from 2883584 to 3153920 tid 3 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:227: read oid 11 snap -1 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:227: expect deleted 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 16 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:222: finishing write tid 3 to smithi03385152-30 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:222: finishing write tid 4 to smithi03385152-30 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:222: finishing write tid 5 to smithi03385152-30 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 145 (ObjNum 93 snap 16 seq_num 93) dirty exists 2024-09-21T09:52:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:222: left oid 30 (ObjNum 93 snap 16 seq_num 93) 2024-09-21T09:52:02.572 INFO:tasks.rados.rados.0.smithi033.stdout:226: finishing write tid 1 to smithi03385152-6 2024-09-21T09:52:02.572 INFO:tasks.rados.rados.0.smithi033.stdout:228: seq_num 95 ranges {458752=794624,1253376=409600,1662976=393216} 2024-09-21T09:52:02.577 INFO:tasks.rados.rados.0.smithi033.stdout:228: writing smithi03385152-28 from 458752 to 1253376 tid 1 2024-09-21T09:52:02.580 INFO:tasks.rados.rados.0.smithi033.stdout:228: writing smithi03385152-28 from 1253376 to 1662976 tid 2 2024-09-21T09:52:02.582 INFO:tasks.rados.rados.0.smithi033.stdout:228: writing smithi03385152-28 from 1662976 to 2056192 tid 3 2024-09-21T09:52:02.582 INFO:tasks.rados.rados.0.smithi033.stdout:222: done (3 left) 2024-09-21T09:52:02.582 INFO:tasks.rados.rados.0.smithi033.stdout:229: snap_create 2024-09-21T09:52:02.583 INFO:tasks.rados.rados.0.smithi033.stdout:226: finishing write tid 2 to smithi03385152-6 2024-09-21T09:52:02.583 INFO:tasks.rados.rados.0.smithi033.stdout:226: finishing write tid 3 to smithi03385152-6 2024-09-21T09:52:02.583 INFO:tasks.rados.rados.0.smithi033.stdout:226: finishing write tid 4 to smithi03385152-6 2024-09-21T09:52:02.583 INFO:tasks.rados.rados.0.smithi033.stdout:226: finishing write tid 5 to smithi03385152-6 2024-09-21T09:52:02.583 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 51 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-09-21T09:52:02.583 INFO:tasks.rados.rados.0.smithi033.stdout:226: left oid 6 (ObjNum 94 snap 16 seq_num 94) 2024-09-21T09:52:02.583 INFO:tasks.rados.rados.0.smithi033.stdout:228: finishing write tid 1 to smithi03385152-28 2024-09-21T09:52:02.583 INFO:tasks.rados.rados.0.smithi033.stdout:228: finishing write tid 2 to smithi03385152-28 2024-09-21T09:52:02.585 INFO:tasks.rados.rados.0.smithi033.stdout:228: finishing write tid 3 to smithi03385152-28 2024-09-21T09:52:02.585 INFO:tasks.rados.rados.0.smithi033.stdout:228: finishing write tid 4 to smithi03385152-28 2024-09-21T09:52:02.589 INFO:tasks.rados.rados.0.smithi033.stdout:228: finishing write tid 5 to smithi03385152-28 2024-09-21T09:52:02.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 24 (ObjNum 95 snap 16 seq_num 95) dirty exists 2024-09-21T09:52:02.589 INFO:tasks.rados.rados.0.smithi033.stdout:228: left oid 28 (ObjNum 95 snap 16 seq_num 95) 2024-09-21T09:52:02.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:02 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:02] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:52:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:02 smithi105 ceph-mon[31286]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:02 smithi105 ceph-mon[31286]: pgmap v153: 121 pgs: 121 active+clean; 50 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.6 MiB/s wr, 31 op/s 2024-09-21T09:52:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:02 smithi105 ceph-mon[31286]: from='client.24817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:02.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:02 smithi105 ceph-mon[31286]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:52:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:02 smithi033 ceph-mon[39045]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:02 smithi033 ceph-mon[39045]: pgmap v153: 121 pgs: 121 active+clean; 50 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.6 MiB/s wr, 31 op/s 2024-09-21T09:52:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:02 smithi033 ceph-mon[39045]: from='client.24817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:02 smithi033 ceph-mon[39045]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:52:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:02 smithi033 ceph-mon[27969]: from='client.14994 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:02 smithi033 ceph-mon[27969]: pgmap v153: 121 pgs: 121 active+clean; 50 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.6 MiB/s wr, 31 op/s 2024-09-21T09:52:02.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:02 smithi033 ceph-mon[27969]: from='client.24817 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:02.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:02 smithi033 ceph-mon[27969]: osdmap e105: 8 total, 8 up, 8 in 2024-09-21T09:52:03.558 INFO:tasks.rados.rados.0.smithi033.stdout:226: done (3 left) 2024-09-21T09:52:03.558 INFO:tasks.rados.rados.0.smithi033.stdout:227: done (2 left) 2024-09-21T09:52:03.558 INFO:tasks.rados.rados.0.smithi033.stdout:228: done (1 left) 2024-09-21T09:52:03.558 INFO:tasks.rados.rados.0.smithi033.stdout:229: done (0 left) 2024-09-21T09:52:03.558 INFO:tasks.rados.rados.0.smithi033.stdout:230: delete oid 41 current snap is 17 2024-09-21T09:52:03.562 INFO:tasks.rados.rados.0.smithi033.stdout:230: done (0 left) 2024-09-21T09:52:03.562 INFO:tasks.rados.rados.0.smithi033.stdout:231: read oid 13 snap -1 2024-09-21T09:52:03.562 INFO:tasks.rados.rados.0.smithi033.stdout:231: expect (ObjNum 85 snap 16 seq_num 85) 2024-09-21T09:52:03.562 INFO:tasks.rados.rados.0.smithi033.stdout:232: setattr oid 26 current snap is 17 2024-09-21T09:52:03.565 INFO:tasks.rados.rados.0.smithi033.stdout:233: copy_from oid 32 from oid 7 current snap is 17 2024-09-21T09:52:03.565 INFO:tasks.rados.rados.0.smithi033.stdout:234: read oid 22 snap -1 2024-09-21T09:52:03.565 INFO:tasks.rados.rados.0.smithi033.stdout:234: expect deleted 2024-09-21T09:52:03.565 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 17 2024-09-21T09:52:03.565 INFO:tasks.rados.rados.0.smithi033.stdout:235: seq_num 97 ranges {1024000=434176,1458176=581632,2039808=540672,2580480=81920} 2024-09-21T09:52:03.574 INFO:tasks.rados.rados.0.smithi033.stdout:235: writing smithi03385152-15 from 1024000 to 1458176 tid 1 2024-09-21T09:52:03.577 INFO:tasks.rados.rados.0.smithi033.stdout:235: writing smithi03385152-15 from 1458176 to 2039808 tid 2 2024-09-21T09:52:03.580 INFO:tasks.rados.rados.0.smithi033.stdout:235: writing smithi03385152-15 from 2039808 to 2580480 tid 3 2024-09-21T09:52:03.581 INFO:tasks.rados.rados.0.smithi033.stdout:235: writing smithi03385152-15 from 2580480 to 2662400 tid 4 2024-09-21T09:52:03.581 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 17 2024-09-21T09:52:03.581 INFO:tasks.rados.rados.0.smithi033.stdout:236: seq_num 98 ranges {2310144=450560,2760704=466944,3227648=737280} 2024-09-21T09:52:03.596 INFO:tasks.rados.rados.0.smithi033.stdout:236: writing smithi03385152-49 from 2310144 to 2760704 tid 1 2024-09-21T09:52:03.599 INFO:tasks.rados.rados.0.smithi033.stdout:236: writing smithi03385152-49 from 2760704 to 3227648 tid 2 2024-09-21T09:52:03.603 INFO:tasks.rados.rados.0.smithi033.stdout:236: writing smithi03385152-49 from 3227648 to 3964928 tid 3 2024-09-21T09:52:03.604 INFO:tasks.rados.rados.0.smithi033.stdout:237: rollback oid 33 current snap is 17 2024-09-21T09:52:03.604 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 16 2024-09-21T09:52:03.604 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 62 (ObjNum 129 snap 0 seq_num 91003880) dirty exists 2024-09-21T09:52:03.604 INFO:tasks.rados.rados.0.smithi033.stdout:231: expect (ObjNum 85 snap 16 seq_num 85) 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:233: finishing copy_from racing read to smithi03385152-32 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:233: finishing copy_from to smithi03385152-32 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 2 (ObjNum 6 snap 0 seq_num 6) dirty exists 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:235: finishing write tid 1 to smithi03385152-15 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:235: finishing write tid 2 to smithi03385152-15 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:231: done (6 left) 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:232: done (5 left) 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:233: done (4 left) 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:234: done (3 left) 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 17 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:235: finishing write tid 3 to smithi03385152-15 2024-09-21T09:52:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:235: finishing write tid 4 to smithi03385152-15 2024-09-21T09:52:03.652 INFO:tasks.rados.rados.0.smithi033.stdout:235: finishing write tid 5 to smithi03385152-15 2024-09-21T09:52:03.652 INFO:tasks.rados.rados.0.smithi033.stdout:238: seq_num 99 ranges {0=466944,466944=802816,1269760=401408} 2024-09-21T09:52:03.653 INFO:tasks.rados.rados.0.smithi033.stdout:238: writing smithi03385152-25 from 0 to 466944 tid 1 2024-09-21T09:52:03.658 INFO:tasks.rados.rados.0.smithi033.stdout:238: writing smithi03385152-25 from 466944 to 1269760 tid 2 2024-09-21T09:52:03.661 INFO:tasks.rados.rados.0.smithi033.stdout:238: writing smithi03385152-25 from 1269760 to 1671168 tid 3 2024-09-21T09:52:03.661 INFO:tasks.rados.rados.0.smithi033.stdout:239: read oid 5 snap -1 2024-09-21T09:52:03.661 INFO:tasks.rados.rados.0.smithi033.stdout:239: expect deleted 2024-09-21T09:52:03.661 INFO:tasks.rados.rados.0.smithi033.stdout:240: rollback oid 28 current snap is 17 2024-09-21T09:52:03.661 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 16 2024-09-21T09:52:03.661 INFO:tasks.rados.rados.0.smithi033.stdout:235: finishing write tid 6 to smithi03385152-15 2024-09-21T09:52:03.661 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 32 (ObjNum 97 snap 17 seq_num 97) dirty exists 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:235: left oid 15 (ObjNum 97 snap 17 seq_num 97) 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:236: finishing write tid 1 to smithi03385152-49 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:236: finishing write tid 2 to smithi03385152-49 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:236: finishing write tid 3 to smithi03385152-49 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:235: done (5 left) 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:241: rmattr oid 41 current snap is 17 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:237: finishing rollback tid 0 to smithi03385152-33 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:237: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:236: finishing write tid 4 to smithi03385152-49 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:241: done (5 left) 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:242: copy_from oid 45 from oid 14 current snap is 17 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:237: finishing rollback tid 2 to smithi03385152-33 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 55 (ObjNum 20 snap 0 seq_num 20) dirty exists 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:236: finishing write tid 5 to smithi03385152-49 2024-09-21T09:52:03.662 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 66 (ObjNum 98 snap 17 seq_num 98) dirty exists 2024-09-21T09:52:03.663 INFO:tasks.rados.rados.0.smithi033.stdout:236: left oid 49 (ObjNum 98 snap 17 seq_num 98) 2024-09-21T09:52:03.663 INFO:tasks.rados.rados.0.smithi033.stdout:238: finishing write tid 1 to smithi03385152-25 2024-09-21T09:52:03.663 INFO:tasks.rados.rados.0.smithi033.stdout:236: done (5 left) 2024-09-21T09:52:03.663 INFO:tasks.rados.rados.0.smithi033.stdout:237: done (4 left) 2024-09-21T09:52:03.663 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 17 2024-09-21T09:52:03.663 INFO:tasks.rados.rados.0.smithi033.stdout:243: seq_num 100 ranges {106496=483328,589824=688128,1277952=516096} 2024-09-21T09:52:03.664 INFO:tasks.rados.rados.0.smithi033.stdout:243: writing smithi03385152-32 from 106496 to 589824 tid 1 2024-09-21T09:52:03.668 INFO:tasks.rados.rados.0.smithi033.stdout:243: writing smithi03385152-32 from 589824 to 1277952 tid 2 2024-09-21T09:52:03.672 INFO:tasks.rados.rados.0.smithi033.stdout:243: writing smithi03385152-32 from 1277952 to 1794048 tid 3 2024-09-21T09:52:03.672 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 17 2024-09-21T09:52:03.672 INFO:tasks.rados.rados.0.smithi033.stdout:244: seq_num 101 ranges {0=499712,499712=573440,1073152=524288,1597440=106496} 2024-09-21T09:52:03.675 INFO:tasks.rados.rados.0.smithi033.stdout:244: writing smithi03385152-11 from 0 to 499712 tid 1 2024-09-21T09:52:03.678 INFO:tasks.rados.rados.0.smithi033.stdout:244: writing smithi03385152-11 from 499712 to 1073152 tid 2 2024-09-21T09:52:03.681 INFO:tasks.rados.rados.0.smithi033.stdout:244: writing smithi03385152-11 from 1073152 to 1597440 tid 3 2024-09-21T09:52:03.682 INFO:tasks.rados.rados.0.smithi033.stdout:244: writing smithi03385152-11 from 1597440 to 1703936 tid 4 2024-09-21T09:52:03.682 INFO:tasks.rados.rados.0.smithi033.stdout:245: read oid 46 snap 16 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:245: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:246: snap_create 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:238: finishing write tid 2 to smithi03385152-25 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:238: finishing write tid 3 to smithi03385152-25 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:238: finishing write tid 4 to smithi03385152-25 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:240: finishing rollback tid 0 to smithi03385152-28 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:240: finishing rollback tid 1 to smithi03385152-28 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:240: finishing rollback tid 2 to smithi03385152-28 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 28 (ObjNum 95 snap 16 seq_num 95) dirty exists 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:238: finishing write tid 5 to smithi03385152-25 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 29 (ObjNum 99 snap 17 seq_num 99) dirty exists 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:238: left oid 25 (ObjNum 99 snap 17 seq_num 99) 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:242: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:242: finishing copy_from to smithi03385152-45 2024-09-21T09:52:03.683 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 24 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:243: finishing write tid 1 to smithi03385152-32 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:243: finishing write tid 2 to smithi03385152-32 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:243: finishing write tid 3 to smithi03385152-32 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:243: finishing write tid 4 to smithi03385152-32 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:243: finishing write tid 5 to smithi03385152-32 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 36 (ObjNum 100 snap 17 seq_num 100) dirty exists 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:243: left oid 32 (ObjNum 100 snap 17 seq_num 100) 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:244: finishing write tid 1 to smithi03385152-11 2024-09-21T09:52:03.684 INFO:tasks.rados.rados.0.smithi033.stdout:244: finishing write tid 2 to smithi03385152-11 2024-09-21T09:52:03.685 INFO:tasks.rados.rados.0.smithi033.stdout:244: finishing write tid 3 to smithi03385152-11 2024-09-21T09:52:03.685 INFO:tasks.rados.rados.0.smithi033.stdout:244: finishing write tid 4 to smithi03385152-11 2024-09-21T09:52:03.685 INFO:tasks.rados.rados.0.smithi033.stdout:244: finishing write tid 5 to smithi03385152-11 2024-09-21T09:52:03.686 INFO:tasks.rados.rados.0.smithi033.stdout:244: finishing write tid 6 to smithi03385152-11 2024-09-21T09:52:03.686 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 51 (ObjNum 101 snap 17 seq_num 101) dirty exists 2024-09-21T09:52:03.686 INFO:tasks.rados.rados.0.smithi033.stdout:244: left oid 11 (ObjNum 101 snap 17 seq_num 101) 2024-09-21T09:52:03.686 INFO:tasks.rados.rados.0.smithi033.stdout:245: expect (ObjNum 45 snap 0 seq_num 45) 2024-09-21T09:52:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:03 smithi033 ceph-mon[27969]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:52:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:03 smithi033 ceph-mon[39045]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:52:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:03 smithi105 ceph-mon[31286]: osdmap e106: 8 total, 8 up, 8 in 2024-09-21T09:52:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:238: done (7 left) 2024-09-21T09:52:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:239: done (6 left) 2024-09-21T09:52:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:240: done (5 left) 2024-09-21T09:52:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:242: done (4 left) 2024-09-21T09:52:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:243: done (3 left) 2024-09-21T09:52:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:244: done (2 left) 2024-09-21T09:52:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:245: done (1 left) 2024-09-21T09:52:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:246: done (0 left) 2024-09-21T09:52:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:247: rmattr oid 29 current snap is 18 2024-09-21T09:52:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:247: done (0 left) 2024-09-21T09:52:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:248: copy_from oid 7 from oid 29 current snap is 18 2024-09-21T09:52:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:249: rollback oid 6 current snap is 18 2024-09-21T09:52:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 16 2024-09-21T09:52:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:250: snap_remove snap 17 2024-09-21T09:52:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:04 smithi033 ceph-mon[39045]: pgmap v156: 121 pgs: 121 active+clean; 50 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.6 MiB/s wr, 31 op/s 2024-09-21T09:52:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:04 smithi033 ceph-mon[39045]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:52:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:04 smithi033 ceph-mon[27969]: pgmap v156: 121 pgs: 121 active+clean; 50 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.6 MiB/s wr, 31 op/s 2024-09-21T09:52:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:04 smithi033 ceph-mon[27969]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:52:05.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:04 smithi105 ceph-mon[31286]: pgmap v156: 121 pgs: 121 active+clean; 50 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.7 MiB/s rd, 2.6 MiB/s wr, 31 op/s 2024-09-21T09:52:05.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:04 smithi105 ceph-mon[31286]: osdmap e107: 8 total, 8 up, 8 in 2024-09-21T09:52:05.569 INFO:tasks.rados.rados.0.smithi033.stdout:250: done (2 left) 2024-09-21T09:52:05.569 INFO:tasks.rados.rados.0.smithi033.stdout:251: snap_create 2024-09-21T09:52:05.569 INFO:tasks.rados.rados.0.smithi033.stdout:248: finishing copy_from to smithi03385152-7 2024-09-21T09:52:05.569 INFO:tasks.rados.rados.0.smithi033.stdout:248: got expected ENOENT (src dne) 2024-09-21T09:52:05.569 INFO:tasks.rados.rados.0.smithi033.stdout:248: finishing copy_from racing read to smithi03385152-7 2024-09-21T09:52:05.570 INFO:tasks.rados.rados.0.smithi033.stdout:249: finishing rollback tid 0 to smithi03385152-6 2024-09-21T09:52:05.570 INFO:tasks.rados.rados.0.smithi033.stdout:249: finishing rollback tid 1 to smithi03385152-6 2024-09-21T09:52:05.570 INFO:tasks.rados.rados.0.smithi033.stdout:249: finishing rollback tid 2 to smithi03385152-6 2024-09-21T09:52:05.570 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 59 (ObjNum 94 snap 16 seq_num 94) dirty exists 2024-09-21T09:52:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:05 smithi033 ceph-mon[39045]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:52:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:05 smithi033 ceph-mon[27969]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:52:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:05 smithi105 ceph-mon[31286]: osdmap e108: 8 total, 8 up, 8 in 2024-09-21T09:52:06.467 INFO:tasks.rados.rados.0.smithi033.stdout:248: done (2 left) 2024-09-21T09:52:06.467 INFO:tasks.rados.rados.0.smithi033.stdout:249: done (1 left) 2024-09-21T09:52:06.467 INFO:tasks.rados.rados.0.smithi033.stdout:251: done (0 left) 2024-09-21T09:52:06.467 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 19 2024-09-21T09:52:06.467 INFO:tasks.rados.rados.0.smithi033.stdout:252: seq_num 102 ranges {1605632=516096,2121728=450560,2572288=729088,3301376=24576} 2024-09-21T09:52:06.478 INFO:tasks.rados.rados.0.smithi033.stdout:252: writing smithi03385152-16 from 1605632 to 2121728 tid 1 2024-09-21T09:52:06.480 INFO:tasks.rados.rados.0.smithi033.stdout:252: writing smithi03385152-16 from 2121728 to 2572288 tid 2 2024-09-21T09:52:06.485 INFO:tasks.rados.rados.0.smithi033.stdout:252: writing smithi03385152-16 from 2572288 to 3301376 tid 3 2024-09-21T09:52:06.486 INFO:tasks.rados.rados.0.smithi033.stdout:252: writing smithi03385152-16 from 3301376 to 3325952 tid 4 2024-09-21T09:52:06.486 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 19 2024-09-21T09:52:06.486 INFO:tasks.rados.rados.0.smithi033.stdout:253: seq_num 103 ranges {286720=532480,819200=737280,1556480=466944} 2024-09-21T09:52:06.490 INFO:tasks.rados.rados.0.smithi033.stdout:253: writing smithi03385152-18 from 286720 to 819200 tid 1 2024-09-21T09:52:06.495 INFO:tasks.rados.rados.0.smithi033.stdout:253: writing smithi03385152-18 from 819200 to 1556480 tid 2 2024-09-21T09:52:06.498 INFO:tasks.rados.rados.0.smithi033.stdout:253: writing smithi03385152-18 from 1556480 to 2023424 tid 3 2024-09-21T09:52:06.498 INFO:tasks.rados.rados.0.smithi033.stdout:254: setattr oid 45 current snap is 19 2024-09-21T09:52:06.498 INFO:tasks.rados.rados.0.smithi033.stdout:252: finishing write tid 1 to smithi03385152-16 2024-09-21T09:52:06.498 INFO:tasks.rados.rados.0.smithi033.stdout:252: finishing write tid 2 to smithi03385152-16 2024-09-21T09:52:06.498 INFO:tasks.rados.rados.0.smithi033.stdout:252: finishing write tid 3 to smithi03385152-16 2024-09-21T09:52:06.498 INFO:tasks.rados.rados.0.smithi033.stdout:252: finishing write tid 4 to smithi03385152-16 2024-09-21T09:52:06.499 INFO:tasks.rados.rados.0.smithi033.stdout:252: finishing write tid 5 to smithi03385152-16 2024-09-21T09:52:06.499 INFO:tasks.rados.rados.0.smithi033.stdout:252: finishing write tid 6 to smithi03385152-16 2024-09-21T09:52:06.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 171 (ObjNum 102 snap 19 seq_num 102) dirty exists 2024-09-21T09:52:06.499 INFO:tasks.rados.rados.0.smithi033.stdout:252: left oid 16 (ObjNum 102 snap 19 seq_num 102) 2024-09-21T09:52:06.499 INFO:tasks.rados.rados.0.smithi033.stdout:253: finishing write tid 1 to smithi03385152-18 2024-09-21T09:52:06.499 INFO:tasks.rados.rados.0.smithi033.stdout:253: finishing write tid 2 to smithi03385152-18 2024-09-21T09:52:06.499 INFO:tasks.rados.rados.0.smithi033.stdout:252: done (2 left) 2024-09-21T09:52:06.499 INFO:tasks.rados.rados.0.smithi033.stdout:255: snap_remove snap 16 2024-09-21T09:52:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:06 smithi033 ceph-mon[39045]: pgmap v159: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T09:52:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:06 smithi033 ceph-mon[39045]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:52:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:06 smithi033 ceph-mon[39045]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:52:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:06 smithi033 ceph-mon[27969]: pgmap v159: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T09:52:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:06 smithi033 ceph-mon[27969]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:52:06.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:06 smithi033 ceph-mon[27969]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:52:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:06 smithi105 ceph-mon[31286]: pgmap v159: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T09:52:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:06 smithi105 ceph-mon[31286]: osdmap e109: 8 total, 8 up, 8 in 2024-09-21T09:52:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:06 smithi105 ceph-mon[31286]: osdmap e110: 8 total, 8 up, 8 in 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:253: finishing write tid 3 to smithi03385152-18 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:253: finishing write tid 4 to smithi03385152-18 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 120 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:253: finishing write tid 5 to smithi03385152-18 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 7 (ObjNum 103 snap 19 seq_num 103) dirty exists 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:253: left oid 18 (ObjNum 103 snap 19 seq_num 103) 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:253: done (2 left) 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:254: done (1 left) 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:255: done (0 left) 2024-09-21T09:52:07.469 INFO:tasks.rados.rados.0.smithi033.stdout:256: setattr oid 40 current snap is 19 2024-09-21T09:52:07.471 INFO:tasks.rados.rados.0.smithi033.stdout:257: snap_create 2024-09-21T09:52:07.477 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 84 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:52:08.474 INFO:tasks.rados.rados.0.smithi033.stdout:256: done (1 left) 2024-09-21T09:52:08.474 INFO:tasks.rados.rados.0.smithi033.stdout:257: done (0 left) 2024-09-21T09:52:08.474 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 20 2024-09-21T09:52:08.474 INFO:tasks.rados.rados.0.smithi033.stdout:258: seq_num 104 ranges {344064=548864,892928=622592,1515520=581632} 2024-09-21T09:52:08.478 INFO:tasks.rados.rados.0.smithi033.stdout:258: writing smithi03385152-33 from 344064 to 892928 tid 1 2024-09-21T09:52:08.481 INFO:tasks.rados.rados.0.smithi033.stdout:258: writing smithi03385152-33 from 892928 to 1515520 tid 2 2024-09-21T09:52:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:258: writing smithi03385152-33 from 1515520 to 2097152 tid 3 2024-09-21T09:52:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:259: copy_from oid 18 from oid 13 current snap is 20 2024-09-21T09:52:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:258: finishing write tid 1 to smithi03385152-33 2024-09-21T09:52:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:258: finishing write tid 2 to smithi03385152-33 2024-09-21T09:52:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:260: snap_create 2024-09-21T09:52:08.487 INFO:tasks.rados.rados.0.smithi033.stdout:258: finishing write tid 3 to smithi03385152-33 2024-09-21T09:52:08.487 INFO:tasks.rados.rados.0.smithi033.stdout:258: finishing write tid 4 to smithi03385152-33 2024-09-21T09:52:08.488 INFO:tasks.rados.rados.0.smithi033.stdout:258: finishing write tid 5 to smithi03385152-33 2024-09-21T09:52:08.488 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 70 (ObjNum 104 snap 20 seq_num 104) dirty exists 2024-09-21T09:52:08.489 INFO:tasks.rados.rados.0.smithi033.stdout:258: left oid 33 (ObjNum 104 snap 20 seq_num 104) 2024-09-21T09:52:08.496 INFO:tasks.rados.rados.0.smithi033.stdout:259: finishing copy_from racing read to smithi03385152-18 2024-09-21T09:52:08.497 INFO:tasks.rados.rados.0.smithi033.stdout:259: finishing copy_from to smithi03385152-18 2024-09-21T09:52:08.497 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 100 (ObjNum 85 snap 16 seq_num 85) dirty exists 2024-09-21T09:52:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:08 smithi033 ceph-mon[39045]: pgmap v162: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T09:52:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:08 smithi033 ceph-mon[39045]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:52:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:08 smithi033 ceph-mon[27969]: pgmap v162: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T09:52:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:08 smithi033 ceph-mon[27969]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:52:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:08 smithi105 ceph-mon[31286]: pgmap v162: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T09:52:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:08 smithi105 ceph-mon[31286]: osdmap e111: 8 total, 8 up, 8 in 2024-09-21T09:52:09.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:08] "GET /metrics HTTP/1.1" 200 239906 "" "Prometheus/2.43.0" 2024-09-21T09:52:09.479 INFO:tasks.rados.rados.0.smithi033.stdout:258: done (2 left) 2024-09-21T09:52:09.479 INFO:tasks.rados.rados.0.smithi033.stdout:259: done (1 left) 2024-09-21T09:52:09.479 INFO:tasks.rados.rados.0.smithi033.stdout:260: done (0 left) 2024-09-21T09:52:09.479 INFO:tasks.rados.rados.0.smithi033.stdout:261: snap_create 2024-09-21T09:52:09.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:09 smithi033 ceph-mon[39045]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:52:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:09 smithi033 ceph-mon[27969]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:52:09.779 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:09 smithi105 ceph-mon[31286]: osdmap e112: 8 total, 8 up, 8 in 2024-09-21T09:52:10.481 INFO:tasks.rados.rados.0.smithi033.stdout:261: done (0 left) 2024-09-21T09:52:10.481 INFO:tasks.rados.rados.0.smithi033.stdout:262: read oid 15 snap -1 2024-09-21T09:52:10.481 INFO:tasks.rados.rados.0.smithi033.stdout:262: expect (ObjNum 97 snap 17 seq_num 97) 2024-09-21T09:52:10.481 INFO:tasks.rados.rados.0.smithi033.stdout:263: read oid 1 snap -1 2024-09-21T09:52:10.481 INFO:tasks.rados.rados.0.smithi033.stdout:263: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:52:10.481 INFO:tasks.rados.rados.0.smithi033.stdout:264: rmattr oid 37 current snap is 22 2024-09-21T09:52:10.481 INFO:tasks.rados.rados.0.smithi033.stdout:265: copy_from oid 22 from oid 47 current snap is 22 2024-09-21T09:52:10.481 INFO:tasks.rados.rados.0.smithi033.stdout:266: read oid 5 snap -1 2024-09-21T09:52:10.482 INFO:tasks.rados.rados.0.smithi033.stdout:266: expect deleted 2024-09-21T09:52:10.482 INFO:tasks.rados.rados.0.smithi033.stdout:267: read oid 18 snap -1 2024-09-21T09:52:10.482 INFO:tasks.rados.rados.0.smithi033.stdout:267: expect (ObjNum 85 snap 16 seq_num 85) 2024-09-21T09:52:10.482 INFO:tasks.rados.rados.0.smithi033.stdout:268: snap_remove snap 21 2024-09-21T09:52:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:10 smithi033 ceph-mon[39045]: pgmap v165: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:10 smithi033 ceph-mon[39045]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:52:10.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:10 smithi033 ceph-mon[27969]: pgmap v165: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:10.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:10 smithi033 ceph-mon[27969]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:52:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:10 smithi105 ceph-mon[31286]: pgmap v165: 121 pgs: 121 active+clean; 52 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:10 smithi105 ceph-mon[31286]: osdmap e113: 8 total, 8 up, 8 in 2024-09-21T09:52:11.500 INFO:tasks.rados.rados.0.smithi033.stdout:268: done (6 left) 2024-09-21T09:52:11.500 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 22 2024-09-21T09:52:11.500 INFO:tasks.rados.rados.0.smithi033.stdout:269: seq_num 105 ranges {221184=565248,786432=507904,1294336=507904,1802240=188416} 2024-09-21T09:52:11.504 INFO:tasks.rados.rados.0.smithi033.stdout:269: writing smithi03385152-14 from 221184 to 786432 tid 1 2024-09-21T09:52:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:269: writing smithi03385152-14 from 786432 to 1294336 tid 2 2024-09-21T09:52:11.509 INFO:tasks.rados.rados.0.smithi033.stdout:269: writing smithi03385152-14 from 1294336 to 1802240 tid 3 2024-09-21T09:52:11.511 INFO:tasks.rados.rados.0.smithi033.stdout:269: writing smithi03385152-14 from 1802240 to 1990656 tid 4 2024-09-21T09:52:11.511 INFO:tasks.rados.rados.0.smithi033.stdout:270: rollback oid 4 current snap is 22 2024-09-21T09:52:11.511 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 4 to 19 2024-09-21T09:52:11.511 INFO:tasks.rados.rados.0.smithi033.stdout:263: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:52:11.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 53 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:52:11.514 INFO:tasks.rados.rados.0.smithi033.stdout:265: finishing copy_from racing read to smithi03385152-22 2024-09-21T09:52:11.514 INFO:tasks.rados.rados.0.smithi033.stdout:267: expect (ObjNum 85 snap 16 seq_num 85) 2024-09-21T09:52:11.560 INFO:tasks.rados.rados.0.smithi033.stdout:265: finishing copy_from to smithi03385152-22 2024-09-21T09:52:11.560 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:52:11.561 INFO:tasks.rados.rados.0.smithi033.stdout:262: expect (ObjNum 97 snap 17 seq_num 97) 2024-09-21T09:52:11.648 INFO:tasks.rados.rados.0.smithi033.stdout:269: finishing write tid 1 to smithi03385152-14 2024-09-21T09:52:11.648 INFO:tasks.rados.rados.0.smithi033.stdout:269: finishing write tid 2 to smithi03385152-14 2024-09-21T09:52:11.648 INFO:tasks.rados.rados.0.smithi033.stdout:262: done (7 left) 2024-09-21T09:52:11.649 INFO:tasks.rados.rados.0.smithi033.stdout:263: done (6 left) 2024-09-21T09:52:11.649 INFO:tasks.rados.rados.0.smithi033.stdout:264: done (5 left) 2024-09-21T09:52:11.649 INFO:tasks.rados.rados.0.smithi033.stdout:265: done (4 left) 2024-09-21T09:52:11.649 INFO:tasks.rados.rados.0.smithi033.stdout:266: done (3 left) 2024-09-21T09:52:11.649 INFO:tasks.rados.rados.0.smithi033.stdout:267: done (2 left) 2024-09-21T09:52:11.649 INFO:tasks.rados.rados.0.smithi033.stdout:271: snap_create 2024-09-21T09:52:11.649 INFO:tasks.rados.rados.0.smithi033.stdout:270: finishing rollback tid 0 to smithi03385152-4 2024-09-21T09:52:11.649 INFO:tasks.rados.rados.0.smithi033.stdout:270: finishing rollback tid 1 to smithi03385152-4 2024-09-21T09:52:11.650 INFO:tasks.rados.rados.0.smithi033.stdout:270: finishing rollback tid 2 to smithi03385152-4 2024-09-21T09:52:11.650 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 25 (ObjNum 91 snap 16 seq_num 91) dirty exists 2024-09-21T09:52:11.650 INFO:tasks.rados.rados.0.smithi033.stdout:269: finishing write tid 3 to smithi03385152-14 2024-09-21T09:52:11.650 INFO:tasks.rados.rados.0.smithi033.stdout:269: finishing write tid 4 to smithi03385152-14 2024-09-21T09:52:11.650 INFO:tasks.rados.rados.0.smithi033.stdout:269: finishing write tid 5 to smithi03385152-14 2024-09-21T09:52:11.650 INFO:tasks.rados.rados.0.smithi033.stdout:269: finishing write tid 6 to smithi03385152-14 2024-09-21T09:52:11.650 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 129 (ObjNum 105 snap 22 seq_num 105) dirty exists 2024-09-21T09:52:11.650 INFO:tasks.rados.rados.0.smithi033.stdout:269: left oid 14 (ObjNum 105 snap 22 seq_num 105) 2024-09-21T09:52:11.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:11 smithi033 ceph-mon[39045]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:52:11.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:11 smithi033 ceph-mon[27969]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:52:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:11 smithi105 ceph-mon[31286]: osdmap e114: 8 total, 8 up, 8 in 2024-09-21T09:52:12.497 INFO:tasks.rados.rados.0.smithi033.stdout:269: done (2 left) 2024-09-21T09:52:12.497 INFO:tasks.rados.rados.0.smithi033.stdout:270: done (1 left) 2024-09-21T09:52:12.498 INFO:tasks.rados.rados.0.smithi033.stdout:271: done (0 left) 2024-09-21T09:52:12.498 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 23 2024-09-21T09:52:12.498 INFO:tasks.rados.rados.0.smithi033.stdout:272: seq_num 106 ranges {1638400=581632,2220032=794624,3014656=409600} 2024-09-21T09:52:12.509 INFO:tasks.rados.rados.0.smithi033.stdout:272: writing smithi03385152-18 from 1638400 to 2220032 tid 1 2024-09-21T09:52:12.514 INFO:tasks.rados.rados.0.smithi033.stdout:272: writing smithi03385152-18 from 2220032 to 3014656 tid 2 2024-09-21T09:52:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:272: writing smithi03385152-18 from 3014656 to 3424256 tid 3 2024-09-21T09:52:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:273: read oid 31 snap -1 2024-09-21T09:52:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:273: expect deleted 2024-09-21T09:52:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:274: read oid 5 snap -1 2024-09-21T09:52:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:274: expect deleted 2024-09-21T09:52:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:275: copy_from oid 21 from oid 32 current snap is 23 2024-09-21T09:52:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:276: snap_create 2024-09-21T09:52:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:272: finishing write tid 1 to smithi03385152-18 2024-09-21T09:52:12.519 INFO:tasks.rados.rados.0.smithi033.stdout:272: finishing write tid 2 to smithi03385152-18 2024-09-21T09:52:12.520 INFO:tasks.rados.rados.0.smithi033.stdout:272: finishing write tid 3 to smithi03385152-18 2024-09-21T09:52:12.520 INFO:tasks.rados.rados.0.smithi033.stdout:272: finishing write tid 4 to smithi03385152-18 2024-09-21T09:52:12.521 INFO:tasks.rados.rados.0.smithi033.stdout:272: finishing write tid 5 to smithi03385152-18 2024-09-21T09:52:12.521 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 104 (ObjNum 106 snap 23 seq_num 106) dirty exists 2024-09-21T09:52:12.521 INFO:tasks.rados.rados.0.smithi033.stdout:272: left oid 18 (ObjNum 106 snap 23 seq_num 106) 2024-09-21T09:52:12.527 INFO:tasks.rados.rados.0.smithi033.stdout:275: finishing copy_from racing read to smithi03385152-21 2024-09-21T09:52:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:275: finishing copy_from to smithi03385152-21 2024-09-21T09:52:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 36 (ObjNum 100 snap 17 seq_num 100) dirty exists 2024-09-21T09:52:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:12 smithi033 ceph-mon[39045]: pgmap v168: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:52:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:12 smithi033 ceph-mon[39045]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:52:12.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:12 smithi033 ceph-mon[27969]: pgmap v168: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:52:12.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:12 smithi033 ceph-mon[27969]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:52:12.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:12 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:12] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:52:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:12 smithi105 ceph-mon[31286]: pgmap v168: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:52:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:12 smithi105 ceph-mon[31286]: osdmap e115: 8 total, 8 up, 8 in 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:272: done (4 left) 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:273: done (3 left) 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:274: done (2 left) 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:275: done (1 left) 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:276: done (0 left) 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:277: read oid 41 snap -1 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:277: expect deleted 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 24 2024-09-21T09:52:13.502 INFO:tasks.rados.rados.0.smithi033.stdout:278: seq_num 107 ranges {3325952=598016,3923968=729088,4653056=466944,5120000=8192} 2024-09-21T09:52:13.523 INFO:tasks.rados.rados.0.smithi033.stdout:278: writing smithi03385152-16 from 3325952 to 3923968 tid 1 2024-09-21T09:52:13.527 INFO:tasks.rados.rados.0.smithi033.stdout:278: writing smithi03385152-16 from 3923968 to 4653056 tid 2 2024-09-21T09:52:13.531 INFO:tasks.rados.rados.0.smithi033.stdout:278: writing smithi03385152-16 from 4653056 to 5120000 tid 3 2024-09-21T09:52:13.531 INFO:tasks.rados.rados.0.smithi033.stdout:278: writing smithi03385152-16 from 5120000 to 5128192 tid 4 2024-09-21T09:52:13.531 INFO:tasks.rados.rados.0.smithi033.stdout:279: setattr oid 13 current snap is 24 2024-09-21T09:52:13.531 INFO:tasks.rados.rados.0.smithi033.stdout:278: finishing write tid 1 to smithi03385152-16 2024-09-21T09:52:13.531 INFO:tasks.rados.rados.0.smithi033.stdout:278: finishing write tid 2 to smithi03385152-16 2024-09-21T09:52:13.533 INFO:tasks.rados.rados.0.smithi033.stdout:278: finishing write tid 3 to smithi03385152-16 2024-09-21T09:52:13.533 INFO:tasks.rados.rados.0.smithi033.stdout:278: finishing write tid 4 to smithi03385152-16 2024-09-21T09:52:13.534 INFO:tasks.rados.rados.0.smithi033.stdout:278: finishing write tid 5 to smithi03385152-16 2024-09-21T09:52:13.534 INFO:tasks.rados.rados.0.smithi033.stdout:277: done (2 left) 2024-09-21T09:52:13.534 INFO:tasks.rados.rados.0.smithi033.stdout:280: delete oid 37 current snap is 24 2024-09-21T09:52:13.536 INFO:tasks.rados.rados.0.smithi033.stdout:278: finishing write tid 6 to smithi03385152-16 2024-09-21T09:52:13.536 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 179 (ObjNum 107 snap 24 seq_num 107) dirty exists 2024-09-21T09:52:13.536 INFO:tasks.rados.rados.0.smithi033.stdout:278: left oid 16 (ObjNum 107 snap 24 seq_num 107) 2024-09-21T09:52:13.536 INFO:tasks.rados.rados.0.smithi033.stdout:278: done (2 left) 2024-09-21T09:52:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:280: done (1 left) 2024-09-21T09:52:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:281: read oid 21 snap -1 2024-09-21T09:52:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:281: expect (ObjNum 100 snap 17 seq_num 100) 2024-09-21T09:52:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:282: read oid 28 snap 18 2024-09-21T09:52:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:282: expect (ObjNum 95 snap 16 seq_num 95) 2024-09-21T09:52:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 24 2024-09-21T09:52:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:283: seq_num 109 ranges {3153920=630784,3784704=491520,4276224=483328,4759552=229376} 2024-09-21T09:52:13.558 INFO:tasks.rados.rados.0.smithi033.stdout:283: writing smithi03385152-6 from 3153920 to 3784704 tid 1 2024-09-21T09:52:13.560 INFO:tasks.rados.rados.0.smithi033.stdout:283: writing smithi03385152-6 from 3784704 to 4276224 tid 2 2024-09-21T09:52:13.563 INFO:tasks.rados.rados.0.smithi033.stdout:283: writing smithi03385152-6 from 4276224 to 4759552 tid 3 2024-09-21T09:52:13.564 INFO:tasks.rados.rados.0.smithi033.stdout:283: writing smithi03385152-6 from 4759552 to 4988928 tid 4 2024-09-21T09:52:13.564 INFO:tasks.rados.rados.0.smithi033.stdout:284: rollback oid 15 current snap is 24 2024-09-21T09:52:13.564 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 15 to 20 2024-09-21T09:52:13.564 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 181 (ObjNum 85 snap 16 seq_num 85) dirty exists 2024-09-21T09:52:13.565 INFO:tasks.rados.rados.0.smithi033.stdout:281: expect (ObjNum 100 snap 17 seq_num 100) 2024-09-21T09:52:13.615 INFO:tasks.rados.rados.0.smithi033.stdout:282: expect (ObjNum 95 snap 16 seq_num 95) 2024-09-21T09:52:13.680 INFO:tasks.rados.rados.0.smithi033.stdout:283: finishing write tid 1 to smithi03385152-6 2024-09-21T09:52:13.680 INFO:tasks.rados.rados.0.smithi033.stdout:283: finishing write tid 2 to smithi03385152-6 2024-09-21T09:52:13.680 INFO:tasks.rados.rados.0.smithi033.stdout:283: finishing write tid 3 to smithi03385152-6 2024-09-21T09:52:13.680 INFO:tasks.rados.rados.0.smithi033.stdout:283: finishing write tid 4 to smithi03385152-6 2024-09-21T09:52:13.680 INFO:tasks.rados.rados.0.smithi033.stdout:284: finishing rollback tid 0 to smithi03385152-15 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:284: finishing rollback tid 1 to smithi03385152-15 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:279: done (4 left) 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:281: done (3 left) 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:282: done (2 left) 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:285: read oid 47 snap -1 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:285: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:286: delete oid 20 current snap is 24 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:284: finishing rollback tid 2 to smithi03385152-15 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 38 (ObjNum 97 snap 17 seq_num 97) dirty exists 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:283: finishing write tid 5 to smithi03385152-6 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:283: finishing write tid 6 to smithi03385152-6 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 76 (ObjNum 109 snap 24 seq_num 109) dirty exists 2024-09-21T09:52:13.681 INFO:tasks.rados.rados.0.smithi033.stdout:283: left oid 6 (ObjNum 109 snap 24 seq_num 109) 2024-09-21T09:52:13.682 INFO:tasks.rados.rados.0.smithi033.stdout:283: done (3 left) 2024-09-21T09:52:13.682 INFO:tasks.rados.rados.0.smithi033.stdout:284: done (2 left) 2024-09-21T09:52:13.682 INFO:tasks.rados.rados.0.smithi033.stdout:286: done (1 left) 2024-09-21T09:52:13.682 INFO:tasks.rados.rados.0.smithi033.stdout:287: rollback oid 26 current snap is 24 2024-09-21T09:52:13.682 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 26 to 18 2024-09-21T09:52:13.682 INFO:tasks.rados.rados.0.smithi033.stdout:288: rollback oid 37 current snap is 24 2024-09-21T09:52:13.682 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 19 2024-09-21T09:52:13.682 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 24 2024-09-21T09:52:13.683 INFO:tasks.rados.rados.0.smithi033.stdout:289: seq_num 111 ranges {1990656=663552,2654208=663552,3317760=450560,3768320=90112} 2024-09-21T09:52:13.696 INFO:tasks.rados.rados.0.smithi033.stdout:289: writing smithi03385152-14 from 1990656 to 2654208 tid 1 2024-09-21T09:52:13.700 INFO:tasks.rados.rados.0.smithi033.stdout:289: writing smithi03385152-14 from 2654208 to 3317760 tid 2 2024-09-21T09:52:13.702 INFO:tasks.rados.rados.0.smithi033.stdout:289: writing smithi03385152-14 from 3317760 to 3768320 tid 3 2024-09-21T09:52:13.704 INFO:tasks.rados.rados.0.smithi033.stdout:289: writing smithi03385152-14 from 3768320 to 3858432 tid 4 2024-09-21T09:52:13.704 INFO:tasks.rados.rados.0.smithi033.stdout:290: read oid 5 snap -1 2024-09-21T09:52:13.704 INFO:tasks.rados.rados.0.smithi033.stdout:290: expect deleted 2024-09-21T09:52:13.704 INFO:tasks.rados.rados.0.smithi033.stdout:291: rollback oid 36 current snap is 24 2024-09-21T09:52:13.704 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 18 2024-09-21T09:52:13.704 INFO:tasks.rados.rados.0.smithi033.stdout:285: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:52:13.739 INFO:tasks.rados.rados.0.smithi033.stdout:288: finishing rollback tid 1 to smithi03385152-37 2024-09-21T09:52:13.739 INFO:tasks.rados.rados.0.smithi033.stdout:287: finishing rollback tid 0 to smithi03385152-26 2024-09-21T09:52:13.739 INFO:tasks.rados.rados.0.smithi033.stdout:288: finishing rollback tid 2 to smithi03385152-37 2024-09-21T09:52:13.739 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 59 (ObjNum 26 snap 0 seq_num 26) dirty exists 2024-09-21T09:52:13.739 INFO:tasks.rados.rados.0.smithi033.stdout:287: finishing rollback tid 1 to smithi03385152-26 2024-09-21T09:52:13.739 INFO:tasks.rados.rados.0.smithi033.stdout:287: finishing rollback tid 2 to smithi03385152-26 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 71 (ObjNum 65 snap 0 seq_num 1701469276) dirty exists 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:289: finishing write tid 1 to smithi03385152-14 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:289: finishing write tid 2 to smithi03385152-14 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:285: done (5 left) 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:287: done (4 left) 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:288: done (3 left) 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:292: snap_create 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:289: finishing write tid 3 to smithi03385152-14 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:289: finishing write tid 4 to smithi03385152-14 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:289: finishing write tid 5 to smithi03385152-14 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:291: finishing rollback tid 1 to smithi03385152-36 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 0 (ObjNum 0 snap 0 seq_num 304) dirty dne 2024-09-21T09:52:13.740 INFO:tasks.rados.rados.0.smithi033.stdout:289: finishing write tid 6 to smithi03385152-14 2024-09-21T09:52:13.741 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 135 (ObjNum 111 snap 24 seq_num 111) dirty exists 2024-09-21T09:52:13.741 INFO:tasks.rados.rados.0.smithi033.stdout:289: left oid 14 (ObjNum 111 snap 24 seq_num 111) 2024-09-21T09:52:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:13 smithi105 ceph-mon[31286]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:52:13.970 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:13 smithi033 ceph-mon[27969]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:52:13.970 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:13 smithi033 ceph-mon[39045]: osdmap e116: 8 total, 8 up, 8 in 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:289: done (3 left) 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:290: done (2 left) 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:291: done (1 left) 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:292: done (0 left) 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:293: rollback oid 50 current snap is 25 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 50 to 20 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:294: rmattr oid 38 current snap is 25 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:294: done (1 left) 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 25 2024-09-21T09:52:14.507 INFO:tasks.rados.rados.0.smithi033.stdout:295: seq_num 112 ranges {1794048=679936,2473984=548864,3022848=655360} 2024-09-21T09:52:14.520 INFO:tasks.rados.rados.0.smithi033.stdout:295: writing smithi03385152-21 from 1794048 to 2473984 tid 1 2024-09-21T09:52:14.524 INFO:tasks.rados.rados.0.smithi033.stdout:295: writing smithi03385152-21 from 2473984 to 3022848 tid 2 2024-09-21T09:52:14.528 INFO:tasks.rados.rados.0.smithi033.stdout:295: writing smithi03385152-21 from 3022848 to 3678208 tid 3 2024-09-21T09:52:14.528 INFO:tasks.rados.rados.0.smithi033.stdout:296: copy_from oid 37 from oid 14 current snap is 25 2024-09-21T09:52:14.528 INFO:tasks.rados.rados.0.smithi033.stdout:293: finishing rollback tid 0 to smithi03385152-50 2024-09-21T09:52:14.528 INFO:tasks.rados.rados.0.smithi033.stdout:293: finishing rollback tid 1 to smithi03385152-50 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:293: finishing rollback tid 2 to smithi03385152-50 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 33 (ObjNum 49 snap 0 seq_num 49) dirty exists 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:295: finishing write tid 1 to smithi03385152-21 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:293: done (2 left) 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:297: rmattr oid 44 current snap is 25 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:297: done (2 left) 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 25 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:295: finishing write tid 2 to smithi03385152-21 2024-09-21T09:52:14.529 INFO:tasks.rados.rados.0.smithi033.stdout:298: seq_num 113 ranges {2662400=696320,3358720=434176,3792896=466944,4259840=303104} 2024-09-21T09:52:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:298: writing smithi03385152-15 from 2662400 to 3358720 tid 1 2024-09-21T09:52:14.550 INFO:tasks.rados.rados.0.smithi033.stdout:298: writing smithi03385152-15 from 3358720 to 3792896 tid 2 2024-09-21T09:52:14.552 INFO:tasks.rados.rados.0.smithi033.stdout:298: writing smithi03385152-15 from 3792896 to 4259840 tid 3 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:298: writing smithi03385152-15 from 4259840 to 4562944 tid 4 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 25 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:295: finishing write tid 3 to smithi03385152-21 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:295: finishing write tid 4 to smithi03385152-21 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:295: finishing write tid 5 to smithi03385152-21 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 107 (ObjNum 112 snap 25 seq_num 112) dirty exists 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:295: left oid 21 (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:296: finishing copy_from racing read to smithi03385152-37 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:296: finishing copy_from to smithi03385152-37 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 135 (ObjNum 111 snap 24 seq_num 111) dirty exists 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:298: finishing write tid 1 to smithi03385152-15 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:298: finishing write tid 2 to smithi03385152-15 2024-09-21T09:52:14.555 INFO:tasks.rados.rados.0.smithi033.stdout:299: seq_num 114 ranges {778240=712704,1490944=712704,2203648=491520} 2024-09-21T09:52:14.562 INFO:tasks.rados.rados.0.smithi033.stdout:299: writing smithi03385152-22 from 778240 to 1490944 tid 1 2024-09-21T09:52:14.566 INFO:tasks.rados.rados.0.smithi033.stdout:299: writing smithi03385152-22 from 1490944 to 2203648 tid 2 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:299: writing smithi03385152-22 from 2203648 to 2695168 tid 3 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:295: done (3 left) 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:296: done (2 left) 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:300: copy_from oid 27 from oid 35 current snap is 25 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:298: finishing write tid 3 to smithi03385152-15 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:298: finishing write tid 4 to smithi03385152-15 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:298: finishing write tid 5 to smithi03385152-15 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 25 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:298: finishing write tid 6 to smithi03385152-15 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 44 (ObjNum 113 snap 25 seq_num 113) dirty exists 2024-09-21T09:52:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:298: left oid 15 (ObjNum 113 snap 25 seq_num 113) 2024-09-21T09:52:14.571 INFO:tasks.rados.rados.0.smithi033.stdout:299: finishing write tid 1 to smithi03385152-22 2024-09-21T09:52:14.571 INFO:tasks.rados.rados.0.smithi033.stdout:299: finishing write tid 2 to smithi03385152-22 2024-09-21T09:52:14.571 INFO:tasks.rados.rados.0.smithi033.stdout:301: seq_num 115 ranges {0=729088,729088=647168,1376256=434176,1810432=122880} 2024-09-21T09:52:14.573 INFO:tasks.rados.rados.0.smithi033.stdout:301: writing smithi03385152-31 from 0 to 729088 tid 1 2024-09-21T09:52:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:301: writing smithi03385152-31 from 729088 to 1376256 tid 2 2024-09-21T09:52:14.579 INFO:tasks.rados.rados.0.smithi033.stdout:301: writing smithi03385152-31 from 1376256 to 1810432 tid 3 2024-09-21T09:52:14.580 INFO:tasks.rados.rados.0.smithi033.stdout:301: writing smithi03385152-31 from 1810432 to 1933312 tid 4 2024-09-21T09:52:14.580 INFO:tasks.rados.rados.0.smithi033.stdout:298: done (3 left) 2024-09-21T09:52:14.580 INFO:tasks.rados.rados.0.smithi033.stdout:302: snap_remove snap 18 2024-09-21T09:52:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:14 smithi105 ceph-mon[31286]: pgmap v171: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:52:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:14 smithi105 ceph-mon[31286]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:52:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:14 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:14 smithi033 ceph-mon[27969]: pgmap v171: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:52:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:14 smithi033 ceph-mon[27969]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:52:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:14 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:14 smithi033 ceph-mon[39045]: pgmap v171: 121 pgs: 121 active+clean; 55 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 14 op/s 2024-09-21T09:52:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:14 smithi033 ceph-mon[39045]: osdmap e117: 8 total, 8 up, 8 in 2024-09-21T09:52:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:14 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:15.515 INFO:tasks.rados.rados.0.smithi033.stdout:302: done (3 left) 2024-09-21T09:52:15.515 INFO:tasks.rados.rados.0.smithi033.stdout:303: copy_from oid 15 from oid 44 current snap is 25 2024-09-21T09:52:15.515 INFO:tasks.rados.rados.0.smithi033.stdout:304: copy_from oid 19 from oid 46 current snap is 25 2024-09-21T09:52:15.515 INFO:tasks.rados.rados.0.smithi033.stdout:299: finishing write tid 3 to smithi03385152-22 2024-09-21T09:52:15.515 INFO:tasks.rados.rados.0.smithi033.stdout:299: finishing write tid 4 to smithi03385152-22 2024-09-21T09:52:15.515 INFO:tasks.rados.rados.0.smithi033.stdout:299: finishing write tid 5 to smithi03385152-22 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 140 (ObjNum 114 snap 25 seq_num 114) dirty exists 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:299: left oid 22 (ObjNum 114 snap 25 seq_num 114) 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:301: finishing write tid 1 to smithi03385152-31 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:300: finishing copy_from racing read to smithi03385152-27 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:300: finishing copy_from to smithi03385152-27 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 46 (ObjNum 68 snap 6 seq_num 68) dirty exists 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:301: finishing write tid 2 to smithi03385152-31 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:301: finishing write tid 3 to smithi03385152-31 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:301: finishing write tid 4 to smithi03385152-31 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:301: finishing write tid 5 to smithi03385152-31 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:301: finishing write tid 6 to smithi03385152-31 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 186 (ObjNum 115 snap 25 seq_num 115) dirty exists 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:301: left oid 31 (ObjNum 115 snap 25 seq_num 115) 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:299: done (4 left) 2024-09-21T09:52:15.516 INFO:tasks.rados.rados.0.smithi033.stdout:300: done (3 left) 2024-09-21T09:52:15.517 INFO:tasks.rados.rados.0.smithi033.stdout:301: done (2 left) 2024-09-21T09:52:15.517 INFO:tasks.rados.rados.0.smithi033.stdout:305: snap_create 2024-09-21T09:52:15.531 INFO:tasks.rados.rados.0.smithi033.stdout:304: finishing copy_from racing read to smithi03385152-19 2024-09-21T09:52:15.532 INFO:tasks.rados.rados.0.smithi033.stdout:304: finishing copy_from to smithi03385152-19 2024-09-21T09:52:15.532 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 20 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:52:15.533 INFO:tasks.rados.rados.0.smithi033.stdout:303: finishing copy_from racing read to smithi03385152-15 2024-09-21T09:52:15.534 INFO:tasks.rados.rados.0.smithi033.stdout:303: finishing copy_from to smithi03385152-15 2024-09-21T09:52:15.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 15 (ObjNum 75 snap 12 seq_num 75) dirty exists 2024-09-21T09:52:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:15 smithi105 ceph-mon[31286]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:52:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:15 smithi033 ceph-mon[27969]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:52:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:15 smithi033 ceph-mon[39045]: osdmap e118: 8 total, 8 up, 8 in 2024-09-21T09:52:16.470 INFO:tasks.rados.rados.0.smithi033.stdout:303: done (2 left) 2024-09-21T09:52:16.470 INFO:tasks.rados.rados.0.smithi033.stdout:304: done (1 left) 2024-09-21T09:52:16.470 INFO:tasks.rados.rados.0.smithi033.stdout:305: done (0 left) 2024-09-21T09:52:16.470 INFO:tasks.rados.rados.0.smithi033.stdout:306: read oid 15 snap -1 2024-09-21T09:52:16.470 INFO:tasks.rados.rados.0.smithi033.stdout:306: expect (ObjNum 75 snap 12 seq_num 75) 2024-09-21T09:52:16.470 INFO:tasks.rados.rados.0.smithi033.stdout:307: copy_from oid 3 from oid 47 current snap is 26 2024-09-21T09:52:16.470 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 26 2024-09-21T09:52:16.471 INFO:tasks.rados.rados.0.smithi033.stdout:308: seq_num 116 ranges {1130496=745472,1875968=532480,2408448=638976,3047424=32768} 2024-09-21T09:52:16.480 INFO:tasks.rados.rados.0.smithi033.stdout:308: writing smithi03385152-38 from 1130496 to 1875968 tid 1 2024-09-21T09:52:16.484 INFO:tasks.rados.rados.0.smithi033.stdout:308: writing smithi03385152-38 from 1875968 to 2408448 tid 2 2024-09-21T09:52:16.488 INFO:tasks.rados.rados.0.smithi033.stdout:308: writing smithi03385152-38 from 2408448 to 3047424 tid 3 2024-09-21T09:52:16.488 INFO:tasks.rados.rados.0.smithi033.stdout:308: writing smithi03385152-38 from 3047424 to 3080192 tid 4 2024-09-21T09:52:16.488 INFO:tasks.rados.rados.0.smithi033.stdout:309: read oid 39 snap -1 2024-09-21T09:52:16.488 INFO:tasks.rados.rados.0.smithi033.stdout:309: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:52:16.489 INFO:tasks.rados.rados.0.smithi033.stdout:310: rmattr oid 46 current snap is 26 2024-09-21T09:52:16.489 INFO:tasks.rados.rados.0.smithi033.stdout:306: expect (ObjNum 75 snap 12 seq_num 75) 2024-09-21T09:52:16.525 INFO:tasks.rados.rados.0.smithi033.stdout:307: finishing copy_from racing read to smithi03385152-3 2024-09-21T09:52:16.525 INFO:tasks.rados.rados.0.smithi033.stdout:307: finishing copy_from to smithi03385152-3 2024-09-21T09:52:16.525 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:52:16.525 INFO:tasks.rados.rados.0.smithi033.stdout:306: done (4 left) 2024-09-21T09:52:16.525 INFO:tasks.rados.rados.0.smithi033.stdout:307: done (3 left) 2024-09-21T09:52:16.525 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 26 2024-09-21T09:52:16.525 INFO:tasks.rados.rados.0.smithi033.stdout:308: finishing write tid 1 to smithi03385152-38 2024-09-21T09:52:16.525 INFO:tasks.rados.rados.0.smithi033.stdout:308: finishing write tid 2 to smithi03385152-38 2024-09-21T09:52:16.526 INFO:tasks.rados.rados.0.smithi033.stdout:311: seq_num 117 ranges {1179648=770048,1949696=417792,2367488=442368,2809856=344064} 2024-09-21T09:52:16.535 INFO:tasks.rados.rados.0.smithi033.stdout:311: writing smithi03385152-12 from 1179648 to 1949696 tid 1 2024-09-21T09:52:16.538 INFO:tasks.rados.rados.0.smithi033.stdout:311: writing smithi03385152-12 from 1949696 to 2367488 tid 2 2024-09-21T09:52:16.540 INFO:tasks.rados.rados.0.smithi033.stdout:311: writing smithi03385152-12 from 2367488 to 2809856 tid 3 2024-09-21T09:52:16.542 INFO:tasks.rados.rados.0.smithi033.stdout:311: writing smithi03385152-12 from 2809856 to 3153920 tid 4 2024-09-21T09:52:16.542 INFO:tasks.rados.rados.0.smithi033.stdout:312: read oid 26 snap -1 2024-09-21T09:52:16.543 INFO:tasks.rados.rados.0.smithi033.stdout:312: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:52:16.543 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 26 2024-09-21T09:52:16.543 INFO:tasks.rados.rados.0.smithi033.stdout:313: seq_num 118 ranges {0=786432,786432=704512,1490944=499712} 2024-09-21T09:52:16.547 INFO:tasks.rados.rados.0.smithi033.stdout:313: writing smithi03385152-5 from 0 to 786432 tid 1 2024-09-21T09:52:16.551 INFO:tasks.rados.rados.0.smithi033.stdout:313: writing smithi03385152-5 from 786432 to 1490944 tid 2 2024-09-21T09:52:16.554 INFO:tasks.rados.rados.0.smithi033.stdout:313: writing smithi03385152-5 from 1490944 to 1990656 tid 3 2024-09-21T09:52:16.554 INFO:tasks.rados.rados.0.smithi033.stdout:314: rmattr oid 8 current snap is 26 2024-09-21T09:52:16.555 INFO:tasks.rados.rados.0.smithi033.stdout:314: done (6 left) 2024-09-21T09:52:16.555 INFO:tasks.rados.rados.0.smithi033.stdout:315: read oid 48 snap -1 2024-09-21T09:52:16.555 INFO:tasks.rados.rados.0.smithi033.stdout:315: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:52:16.555 INFO:tasks.rados.rados.0.smithi033.stdout:316: snap_create 2024-09-21T09:52:16.555 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 148 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:52:16.555 INFO:tasks.rados.rados.0.smithi033.stdout:309: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:52:16.572 INFO:tasks.rados.rados.0.smithi033.stdout:308: finishing write tid 3 to smithi03385152-38 2024-09-21T09:52:16.572 INFO:tasks.rados.rados.0.smithi033.stdout:308: finishing write tid 4 to smithi03385152-38 2024-09-21T09:52:16.572 INFO:tasks.rados.rados.0.smithi033.stdout:308: finishing write tid 5 to smithi03385152-38 2024-09-21T09:52:16.572 INFO:tasks.rados.rados.0.smithi033.stdout:308: finishing write tid 6 to smithi03385152-38 2024-09-21T09:52:16.572 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 92 (ObjNum 116 snap 26 seq_num 116) dirty exists 2024-09-21T09:52:16.572 INFO:tasks.rados.rados.0.smithi033.stdout:308: left oid 38 (ObjNum 116 snap 26 seq_num 116) 2024-09-21T09:52:16.572 INFO:tasks.rados.rados.0.smithi033.stdout:311: finishing write tid 1 to smithi03385152-12 2024-09-21T09:52:16.572 INFO:tasks.rados.rados.0.smithi033.stdout:311: finishing write tid 2 to smithi03385152-12 2024-09-21T09:52:16.573 INFO:tasks.rados.rados.0.smithi033.stdout:311: finishing write tid 3 to smithi03385152-12 2024-09-21T09:52:16.573 INFO:tasks.rados.rados.0.smithi033.stdout:311: finishing write tid 4 to smithi03385152-12 2024-09-21T09:52:16.573 INFO:tasks.rados.rados.0.smithi033.stdout:311: finishing write tid 5 to smithi03385152-12 2024-09-21T09:52:16.573 INFO:tasks.rados.rados.0.smithi033.stdout:312: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:52:16.576 INFO:tasks.rados.rados.0.smithi033.stdout:311: finishing write tid 6 to smithi03385152-12 2024-09-21T09:52:16.576 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 79 (ObjNum 117 snap 26 seq_num 117) dirty exists 2024-09-21T09:52:16.576 INFO:tasks.rados.rados.0.smithi033.stdout:311: left oid 12 (ObjNum 117 snap 26 seq_num 117) 2024-09-21T09:52:16.576 INFO:tasks.rados.rados.0.smithi033.stdout:313: finishing write tid 1 to smithi03385152-5 2024-09-21T09:52:16.577 INFO:tasks.rados.rados.0.smithi033.stdout:313: finishing write tid 2 to smithi03385152-5 2024-09-21T09:52:16.577 INFO:tasks.rados.rados.0.smithi033.stdout:313: finishing write tid 3 to smithi03385152-5 2024-09-21T09:52:16.577 INFO:tasks.rados.rados.0.smithi033.stdout:313: finishing write tid 4 to smithi03385152-5 2024-09-21T09:52:16.577 INFO:tasks.rados.rados.0.smithi033.stdout:313: finishing write tid 5 to smithi03385152-5 2024-09-21T09:52:16.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 22 (ObjNum 118 snap 26 seq_num 118) dirty exists 2024-09-21T09:52:16.577 INFO:tasks.rados.rados.0.smithi033.stdout:313: left oid 5 (ObjNum 118 snap 26 seq_num 118) 2024-09-21T09:52:16.577 INFO:tasks.rados.rados.0.smithi033.stdout:315: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:52:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:16 smithi105 ceph-mon[31286]: pgmap v174: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.9 MiB/s wr, 29 op/s 2024-09-21T09:52:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:16 smithi105 ceph-mon[31286]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:52:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:16 smithi105 ceph-mon[31286]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:52:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:16 smithi033 ceph-mon[27969]: pgmap v174: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.9 MiB/s wr, 29 op/s 2024-09-21T09:52:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:16 smithi033 ceph-mon[27969]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:52:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:16 smithi033 ceph-mon[27969]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:52:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:16 smithi033 ceph-mon[39045]: pgmap v174: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.9 MiB/s wr, 29 op/s 2024-09-21T09:52:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:16 smithi033 ceph-mon[39045]: osdmap e119: 8 total, 8 up, 8 in 2024-09-21T09:52:16.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:16 smithi033 ceph-mon[39045]: osdmap e120: 8 total, 8 up, 8 in 2024-09-21T09:52:17.522 INFO:tasks.rados.rados.0.smithi033.stdout:308: done (7 left) 2024-09-21T09:52:17.522 INFO:tasks.rados.rados.0.smithi033.stdout:309: done (6 left) 2024-09-21T09:52:17.522 INFO:tasks.rados.rados.0.smithi033.stdout:310: done (5 left) 2024-09-21T09:52:17.522 INFO:tasks.rados.rados.0.smithi033.stdout:311: done (4 left) 2024-09-21T09:52:17.522 INFO:tasks.rados.rados.0.smithi033.stdout:312: done (3 left) 2024-09-21T09:52:17.522 INFO:tasks.rados.rados.0.smithi033.stdout:313: done (2 left) 2024-09-21T09:52:17.522 INFO:tasks.rados.rados.0.smithi033.stdout:315: done (1 left) 2024-09-21T09:52:17.523 INFO:tasks.rados.rados.0.smithi033.stdout:316: done (0 left) 2024-09-21T09:52:17.523 INFO:tasks.rados.rados.0.smithi033.stdout:317: delete oid 49 current snap is 27 2024-09-21T09:52:17.526 INFO:tasks.rados.rados.0.smithi033.stdout:317: done (0 left) 2024-09-21T09:52:17.526 INFO:tasks.rados.rados.0.smithi033.stdout:318: rollback oid 18 current snap is 27 2024-09-21T09:52:17.526 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 26 2024-09-21T09:52:17.526 INFO:tasks.rados.rados.0.smithi033.stdout:319: snap_create 2024-09-21T09:52:17.529 INFO:tasks.rados.rados.0.smithi033.stdout:318: finishing rollback tid 0 to smithi03385152-18 2024-09-21T09:52:17.529 INFO:tasks.rados.rados.0.smithi033.stdout:318: finishing rollback tid 1 to smithi03385152-18 2024-09-21T09:52:17.529 INFO:tasks.rados.rados.0.smithi033.stdout:318: finishing rollback tid 2 to smithi03385152-18 2024-09-21T09:52:17.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 107 (ObjNum 106 snap 23 seq_num 106) dirty exists 2024-09-21T09:52:18.525 INFO:tasks.rados.rados.0.smithi033.stdout:318: done (1 left) 2024-09-21T09:52:18.525 INFO:tasks.rados.rados.0.smithi033.stdout:319: done (0 left) 2024-09-21T09:52:18.525 INFO:tasks.rados.rados.0.smithi033.stdout:320: rollback oid 41 current snap is 28 2024-09-21T09:52:18.525 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 24 2024-09-21T09:52:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:321: rollback oid 18 current snap is 28 2024-09-21T09:52:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 20 2024-09-21T09:52:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:322: rmattr oid 23 current snap is 28 2024-09-21T09:52:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:323: copy_from oid 13 from oid 37 current snap is 28 2024-09-21T09:52:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 28 2024-09-21T09:52:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:324: seq_num 120 ranges {0=417792,417792=475136,892928=622592,1515520=507904} 2024-09-21T09:52:18.528 INFO:tasks.rados.rados.0.smithi033.stdout:324: writing smithi03385152-36 from 0 to 417792 tid 1 2024-09-21T09:52:18.532 INFO:tasks.rados.rados.0.smithi033.stdout:324: writing smithi03385152-36 from 417792 to 892928 tid 2 2024-09-21T09:52:18.536 INFO:tasks.rados.rados.0.smithi033.stdout:324: writing smithi03385152-36 from 892928 to 1515520 tid 3 2024-09-21T09:52:18.539 INFO:tasks.rados.rados.0.smithi033.stdout:324: writing smithi03385152-36 from 1515520 to 2023424 tid 4 2024-09-21T09:52:18.539 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 28 2024-09-21T09:52:18.539 INFO:tasks.rados.rados.0.smithi033.stdout:325: seq_num 121 ranges {1933312=434176,2367488=753664,3121152=425984,3547136=425984} 2024-09-21T09:52:18.553 INFO:tasks.rados.rados.0.smithi033.stdout:325: writing smithi03385152-31 from 1933312 to 2367488 tid 1 2024-09-21T09:52:18.557 INFO:tasks.rados.rados.0.smithi033.stdout:325: writing smithi03385152-31 from 2367488 to 3121152 tid 2 2024-09-21T09:52:18.559 INFO:tasks.rados.rados.0.smithi033.stdout:325: writing smithi03385152-31 from 3121152 to 3547136 tid 3 2024-09-21T09:52:18.563 INFO:tasks.rados.rados.0.smithi033.stdout:325: writing smithi03385152-31 from 3547136 to 3973120 tid 4 2024-09-21T09:52:18.563 INFO:tasks.rados.rados.0.smithi033.stdout:326: setattr oid 39 current snap is 28 2024-09-21T09:52:18.563 INFO:tasks.rados.rados.0.smithi033.stdout:320: finishing rollback tid 1 to smithi03385152-41 2024-09-21T09:52:18.563 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:52:18.563 INFO:tasks.rados.rados.0.smithi033.stdout:321: finishing rollback tid 0 to smithi03385152-18 2024-09-21T09:52:18.563 INFO:tasks.rados.rados.0.smithi033.stdout:321: finishing rollback tid 1 to smithi03385152-18 2024-09-21T09:52:18.563 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 113 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:52:18.563 INFO:tasks.rados.rados.0.smithi033.stdout:321: finishing rollback tid 2 to smithi03385152-18 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 110 (ObjNum 85 snap 16 seq_num 85) dirty exists 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:324: finishing write tid 1 to smithi03385152-36 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:324: finishing write tid 2 to smithi03385152-36 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:324: finishing write tid 3 to smithi03385152-36 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:324: finishing write tid 4 to smithi03385152-36 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:324: finishing write tid 5 to smithi03385152-36 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:324: finishing write tid 6 to smithi03385152-36 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 195 (ObjNum 120 snap 28 seq_num 120) dirty exists 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:324: left oid 36 (ObjNum 120 snap 28 seq_num 120) 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:323: finishing copy_from racing read to smithi03385152-13 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:323: finishing copy_from to smithi03385152-13 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 135 (ObjNum 111 snap 24 seq_num 111) dirty exists 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:325: finishing write tid 1 to smithi03385152-31 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:325: finishing write tid 2 to smithi03385152-31 2024-09-21T09:52:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:325: finishing write tid 3 to smithi03385152-31 2024-09-21T09:52:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:320: done (6 left) 2024-09-21T09:52:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:321: done (5 left) 2024-09-21T09:52:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:322: done (4 left) 2024-09-21T09:52:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:323: done (3 left) 2024-09-21T09:52:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:324: done (2 left) 2024-09-21T09:52:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:327: snap_create 2024-09-21T09:52:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:325: finishing write tid 4 to smithi03385152-31 2024-09-21T09:52:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:325: finishing write tid 5 to smithi03385152-31 2024-09-21T09:52:18.566 INFO:tasks.rados.rados.0.smithi033.stdout:325: finishing write tid 6 to smithi03385152-31 2024-09-21T09:52:18.566 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 203 (ObjNum 121 snap 28 seq_num 121) dirty exists 2024-09-21T09:52:18.566 INFO:tasks.rados.rados.0.smithi033.stdout:325: left oid 31 (ObjNum 121 snap 28 seq_num 121) 2024-09-21T09:52:18.567 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 24 (ObjNum 38 snap 0 seq_num 38) dirty exists 2024-09-21T09:52:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:18 smithi105 ceph-mon[31286]: pgmap v177: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.9 MiB/s wr, 29 op/s 2024-09-21T09:52:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:18 smithi105 ceph-mon[31286]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:52:18.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:18 smithi033 ceph-mon[39045]: pgmap v177: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.9 MiB/s wr, 29 op/s 2024-09-21T09:52:18.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:18 smithi033 ceph-mon[39045]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:52:18.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:18 smithi033 ceph-mon[27969]: pgmap v177: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.4 MiB/s rd, 5.9 MiB/s wr, 29 op/s 2024-09-21T09:52:18.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:18 smithi033 ceph-mon[27969]: osdmap e121: 8 total, 8 up, 8 in 2024-09-21T09:52:19.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:18] "GET /metrics HTTP/1.1" 200 240011 "" "Prometheus/2.43.0" 2024-09-21T09:52:19.532 INFO:tasks.rados.rados.0.smithi033.stdout:325: done (2 left) 2024-09-21T09:52:19.532 INFO:tasks.rados.rados.0.smithi033.stdout:326: done (1 left) 2024-09-21T09:52:19.532 INFO:tasks.rados.rados.0.smithi033.stdout:327: done (0 left) 2024-09-21T09:52:19.532 INFO:tasks.rados.rados.0.smithi033.stdout:328: copy_from oid 34 from oid 46 current snap is 29 2024-09-21T09:52:19.532 INFO:tasks.rados.rados.0.smithi033.stdout:329: snap_remove snap 26 2024-09-21T09:52:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:19 smithi105 ceph-mon[31286]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:52:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:19 smithi033 ceph-mon[27969]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:52:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:19 smithi033 ceph-mon[39045]: osdmap e122: 8 total, 8 up, 8 in 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:329: done (1 left) 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:330: rollback oid 45 current snap is 29 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 45 to 23 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:331: read oid 20 snap -1 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:331: expect deleted 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:332: read oid 37 snap -1 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:332: expect (ObjNum 111 snap 24 seq_num 111) 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:333: delete oid 28 current snap is 29 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:328: finishing copy_from racing read to smithi03385152-34 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:328: finishing copy_from to smithi03385152-34 2024-09-21T09:52:20.539 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 148 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:52:20.549 INFO:tasks.rados.rados.0.smithi033.stdout:328: done (4 left) 2024-09-21T09:52:20.549 INFO:tasks.rados.rados.0.smithi033.stdout:331: done (3 left) 2024-09-21T09:52:20.549 INFO:tasks.rados.rados.0.smithi033.stdout:333: done (2 left) 2024-09-21T09:52:20.549 INFO:tasks.rados.rados.0.smithi033.stdout:334: rollback oid 13 current snap is 29 2024-09-21T09:52:20.549 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 13 to 25 2024-09-21T09:52:20.549 INFO:tasks.rados.rados.0.smithi033.stdout:335: snap_remove snap 28 2024-09-21T09:52:20.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:20 smithi105 ceph-mon[31286]: pgmap v180: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:20.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:20 smithi105 ceph-mon[31286]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:52:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:20 smithi033 ceph-mon[27969]: pgmap v180: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:20 smithi033 ceph-mon[27969]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:52:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:20 smithi033 ceph-mon[39045]: pgmap v180: 121 pgs: 121 active+clean; 79 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:20 smithi033 ceph-mon[39045]: osdmap e123: 8 total, 8 up, 8 in 2024-09-21T09:52:21.549 INFO:tasks.rados.rados.0.smithi033.stdout:330: finishing rollback tid 0 to smithi03385152-45 2024-09-21T09:52:21.549 INFO:tasks.rados.rados.0.smithi033.stdout:330: finishing rollback tid 1 to smithi03385152-45 2024-09-21T09:52:21.549 INFO:tasks.rados.rados.0.smithi033.stdout:330: finishing rollback tid 2 to smithi03385152-45 2024-09-21T09:52:21.550 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 154 (ObjNum 13 snap 0 seq_num 13) dirty exists 2024-09-21T09:52:21.550 INFO:tasks.rados.rados.0.smithi033.stdout:334: finishing rollback tid 0 to smithi03385152-13 2024-09-21T09:52:21.550 INFO:tasks.rados.rados.0.smithi033.stdout:334: finishing rollback tid 1 to smithi03385152-13 2024-09-21T09:52:21.550 INFO:tasks.rados.rados.0.smithi033.stdout:332: expect (ObjNum 111 snap 24 seq_num 111) 2024-09-21T09:52:21.687 INFO:tasks.rados.rados.0.smithi033.stdout:334: finishing rollback tid 2 to smithi03385152-13 2024-09-21T09:52:21.687 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 211 (ObjNum 85 snap 16 seq_num 85) dirty exists 2024-09-21T09:52:21.687 INFO:tasks.rados.rados.0.smithi033.stdout:330: done (3 left) 2024-09-21T09:52:21.687 INFO:tasks.rados.rados.0.smithi033.stdout:332: done (2 left) 2024-09-21T09:52:21.687 INFO:tasks.rados.rados.0.smithi033.stdout:334: done (1 left) 2024-09-21T09:52:21.687 INFO:tasks.rados.rados.0.smithi033.stdout:335: done (0 left) 2024-09-21T09:52:21.687 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 29 2024-09-21T09:52:21.687 INFO:tasks.rados.rados.0.smithi033.stdout:336: seq_num 123 ranges {1531904=466944,1998848=573440,2572288=794624,3366912=237568} 2024-09-21T09:52:21.698 INFO:tasks.rados.rados.0.smithi033.stdout:336: writing smithi03385152-4 from 1531904 to 1998848 tid 1 2024-09-21T09:52:21.701 INFO:tasks.rados.rados.0.smithi033.stdout:336: writing smithi03385152-4 from 1998848 to 2572288 tid 2 2024-09-21T09:52:21.705 INFO:tasks.rados.rados.0.smithi033.stdout:336: writing smithi03385152-4 from 2572288 to 3366912 tid 3 2024-09-21T09:52:21.707 INFO:tasks.rados.rados.0.smithi033.stdout:336: writing smithi03385152-4 from 3366912 to 3604480 tid 4 2024-09-21T09:52:21.707 INFO:tasks.rados.rados.0.smithi033.stdout:337: read oid 45 snap -1 2024-09-21T09:52:21.708 INFO:tasks.rados.rados.0.smithi033.stdout:337: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:52:21.708 INFO:tasks.rados.rados.0.smithi033.stdout:338: copy_from oid 21 from oid 20 current snap is 29 2024-09-21T09:52:21.708 INFO:tasks.rados.rados.0.smithi033.stdout:336: finishing write tid 1 to smithi03385152-4 2024-09-21T09:52:21.708 INFO:tasks.rados.rados.0.smithi033.stdout:336: finishing write tid 2 to smithi03385152-4 2024-09-21T09:52:21.708 INFO:tasks.rados.rados.0.smithi033.stdout:339: setattr oid 33 current snap is 29 2024-09-21T09:52:21.709 INFO:tasks.rados.rados.0.smithi033.stdout:338: finishing copy_from to smithi03385152-21 2024-09-21T09:52:21.709 INFO:tasks.rados.rados.0.smithi033.stdout:338: got expected ENOENT (src dne) 2024-09-21T09:52:21.709 INFO:tasks.rados.rados.0.smithi033.stdout:338: finishing copy_from racing read to smithi03385152-21 2024-09-21T09:52:21.709 INFO:tasks.rados.rados.0.smithi033.stdout:338: done (3 left) 2024-09-21T09:52:21.710 INFO:tasks.rados.rados.0.smithi033.stdout:340: read oid 35 snap -1 2024-09-21T09:52:21.710 INFO:tasks.rados.rados.0.smithi033.stdout:340: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:52:21.710 INFO:tasks.rados.rados.0.smithi033.stdout:341: read oid 39 snap -1 2024-09-21T09:52:21.710 INFO:tasks.rados.rados.0.smithi033.stdout:341: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:52:21.710 INFO:tasks.rados.rados.0.smithi033.stdout:342: delete oid 5 current snap is 29 2024-09-21T09:52:21.711 INFO:tasks.rados.rados.0.smithi033.stdout:336: finishing write tid 3 to smithi03385152-4 2024-09-21T09:52:21.711 INFO:tasks.rados.rados.0.smithi033.stdout:336: finishing write tid 4 to smithi03385152-4 2024-09-21T09:52:21.711 INFO:tasks.rados.rados.0.smithi033.stdout:336: finishing write tid 5 to smithi03385152-4 2024-09-21T09:52:21.713 INFO:tasks.rados.rados.0.smithi033.stdout:341: expect (ObjNum 38 snap 0 seq_num 38) 2024-09-21T09:52:21.733 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 86 (ObjNum 104 snap 20 seq_num 104) dirty exists 2024-09-21T09:52:21.734 INFO:tasks.rados.rados.0.smithi033.stdout:337: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:52:21.741 INFO:tasks.rados.rados.0.smithi033.stdout:336: finishing write tid 6 to smithi03385152-4 2024-09-21T09:52:21.741 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 33 (ObjNum 123 snap 29 seq_num 123) dirty exists 2024-09-21T09:52:21.741 INFO:tasks.rados.rados.0.smithi033.stdout:336: left oid 4 (ObjNum 123 snap 29 seq_num 123) 2024-09-21T09:52:21.741 INFO:tasks.rados.rados.0.smithi033.stdout:340: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:52:21.797 INFO:tasks.rados.rados.0.smithi033.stdout:336: done (5 left) 2024-09-21T09:52:21.797 INFO:tasks.rados.rados.0.smithi033.stdout:337: done (4 left) 2024-09-21T09:52:21.797 INFO:tasks.rados.rados.0.smithi033.stdout:339: done (3 left) 2024-09-21T09:52:21.797 INFO:tasks.rados.rados.0.smithi033.stdout:340: done (2 left) 2024-09-21T09:52:21.798 INFO:tasks.rados.rados.0.smithi033.stdout:341: done (1 left) 2024-09-21T09:52:21.798 INFO:tasks.rados.rados.0.smithi033.stdout:342: done (0 left) 2024-09-21T09:52:21.798 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 29 2024-09-21T09:52:21.798 INFO:tasks.rados.rados.0.smithi033.stdout:343: seq_num 125 ranges {3153920=499712,3653632=745472,4399104=409600,4808704=450560} 2024-09-21T09:52:21.818 INFO:tasks.rados.rados.0.smithi033.stdout:343: writing smithi03385152-12 from 3153920 to 3653632 tid 1 2024-09-21T09:52:21.821 INFO:tasks.rados.rados.0.smithi033.stdout:343: writing smithi03385152-12 from 3653632 to 4399104 tid 2 2024-09-21T09:52:21.824 INFO:tasks.rados.rados.0.smithi033.stdout:343: writing smithi03385152-12 from 4399104 to 4808704 tid 3 2024-09-21T09:52:21.827 INFO:tasks.rados.rados.0.smithi033.stdout:343: writing smithi03385152-12 from 4808704 to 5259264 tid 4 2024-09-21T09:52:21.827 INFO:tasks.rados.rados.0.smithi033.stdout:344: snap_remove snap 20 2024-09-21T09:52:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:21 smithi033 ceph-mon[27969]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:52:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:21 smithi033 ceph-mon[39045]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:52:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:21 smithi105 ceph-mon[31286]: osdmap e124: 8 total, 8 up, 8 in 2024-09-21T09:52:22.553 INFO:tasks.rados.rados.0.smithi033.stdout:344: done (1 left) 2024-09-21T09:52:22.554 INFO:tasks.rados.rados.0.smithi033.stdout:345: snap_remove snap 25 2024-09-21T09:52:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:22 smithi105 ceph-mon[31286]: pgmap v183: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-21T09:52:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:22 smithi105 ceph-mon[31286]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:52:22.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[33366]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:52:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:22 smithi033 ceph-mon[39045]: pgmap v183: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-21T09:52:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:22 smithi033 ceph-mon[39045]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:52:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:22 smithi033 ceph-mon[27969]: pgmap v183: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-21T09:52:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:22 smithi033 ceph-mon[27969]: osdmap e125: 8 total, 8 up, 8 in 2024-09-21T09:52:23.562 INFO:tasks.rados.rados.0.smithi033.stdout:345: done (1 left) 2024-09-21T09:52:23.562 INFO:tasks.rados.rados.0.smithi033.stdout:346: snap_remove snap 27 2024-09-21T09:52:23.679 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:23 smithi105 ceph-mon[31286]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:52:23.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:23 smithi033 ceph-mon[27969]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:52:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:23 smithi033 ceph-mon[39045]: osdmap e126: 8 total, 8 up, 8 in 2024-09-21T09:52:24.565 INFO:tasks.rados.rados.0.smithi033.stdout:346: done (1 left) 2024-09-21T09:52:24.565 INFO:tasks.rados.rados.0.smithi033.stdout:347: snap_remove snap 24 2024-09-21T09:52:24.746 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:24 smithi105 ceph-mon[31286]: pgmap v186: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-21T09:52:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:24 smithi105 ceph-mon[31286]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:52:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:52:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:52:24.747 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:24 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[39045]: pgmap v186: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-21T09:52:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[39045]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:52:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:52:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:52:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[27969]: pgmap v186: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-09-21T09:52:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[27969]: osdmap e127: 8 total, 8 up, 8 in 2024-09-21T09:52:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:52:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:52:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:24 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:347: done (1 left) 2024-09-21T09:52:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:348: setattr oid 21 current snap is 29 2024-09-21T09:52:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:343: finishing write tid 1 to smithi03385152-12 2024-09-21T09:52:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:343: finishing write tid 2 to smithi03385152-12 2024-09-21T09:52:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:343: finishing write tid 3 to smithi03385152-12 2024-09-21T09:52:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:343: finishing write tid 4 to smithi03385152-12 2024-09-21T09:52:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:343: finishing write tid 5 to smithi03385152-12 2024-09-21T09:52:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:343: finishing write tid 6 to smithi03385152-12 2024-09-21T09:52:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 89 (ObjNum 125 snap 29 seq_num 125) dirty exists 2024-09-21T09:52:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:343: left oid 12 (ObjNum 125 snap 29 seq_num 125) 2024-09-21T09:52:25.578 INFO:tasks.rados.rados.0.smithi033.stdout:343: done (1 left) 2024-09-21T09:52:25.578 INFO:tasks.rados.rados.0.smithi033.stdout:349: read oid 27 snap -1 2024-09-21T09:52:25.578 INFO:tasks.rados.rados.0.smithi033.stdout:349: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:52:25.578 INFO:tasks.rados.rados.0.smithi033.stdout:350: read oid 5 snap -1 2024-09-21T09:52:25.578 INFO:tasks.rados.rados.0.smithi033.stdout:350: expect deleted 2024-09-21T09:52:25.578 INFO:tasks.rados.rados.0.smithi033.stdout:351: read oid 28 snap 22 2024-09-21T09:52:25.578 INFO:tasks.rados.rados.0.smithi033.stdout:351: expect (ObjNum 95 snap 16 seq_num 95) 2024-09-21T09:52:25.578 INFO:tasks.rados.rados.0.smithi033.stdout:352: copy_from oid 29 from oid 23 current snap is 29 2024-09-21T09:52:25.579 INFO:tasks.rados.rados.0.smithi033.stdout:353: snap_create 2024-09-21T09:52:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 130 (ObjNum 112 snap 25 seq_num 112) dirty exists 2024-09-21T09:52:25.587 INFO:tasks.rados.rados.0.smithi033.stdout:351: expect (ObjNum 95 snap 16 seq_num 95) 2024-09-21T09:52:25.653 INFO:tasks.rados.rados.0.smithi033.stdout:349: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:52:25.710 INFO:tasks.rados.rados.0.smithi033.stdout:352: finishing copy_from racing read to smithi03385152-29 2024-09-21T09:52:25.710 INFO:tasks.rados.rados.0.smithi033.stdout:352: finishing copy_from to smithi03385152-29 2024-09-21T09:52:25.710 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 113 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:52:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:25 smithi105 ceph-mon[31286]: Upgrade: Updating mgr.x 2024-09-21T09:52:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:25 smithi105 ceph-mon[31286]: Deploying daemon mgr.x on smithi105 2024-09-21T09:52:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:25 smithi105 ceph-mon[31286]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:52:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:25 smithi033 ceph-mon[27969]: Upgrade: Updating mgr.x 2024-09-21T09:52:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:25 smithi033 ceph-mon[27969]: Deploying daemon mgr.x on smithi105 2024-09-21T09:52:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:25 smithi033 ceph-mon[27969]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:52:25.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:25 smithi033 ceph-mon[39045]: Upgrade: Updating mgr.x 2024-09-21T09:52:25.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:25 smithi033 ceph-mon[39045]: Deploying daemon mgr.x on smithi105 2024-09-21T09:52:25.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:25 smithi033 ceph-mon[39045]: osdmap e128: 8 total, 8 up, 8 in 2024-09-21T09:52:26.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:26 smithi105 systemd[1]: Stopping Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:52:26.471 INFO:tasks.rados.rados.0.smithi033.stdout:348: done (5 left) 2024-09-21T09:52:26.471 INFO:tasks.rados.rados.0.smithi033.stdout:349: done (4 left) 2024-09-21T09:52:26.471 INFO:tasks.rados.rados.0.smithi033.stdout:350: done (3 left) 2024-09-21T09:52:26.471 INFO:tasks.rados.rados.0.smithi033.stdout:351: done (2 left) 2024-09-21T09:52:26.471 INFO:tasks.rados.rados.0.smithi033.stdout:352: done (1 left) 2024-09-21T09:52:26.472 INFO:tasks.rados.rados.0.smithi033.stdout:353: done (0 left) 2024-09-21T09:52:26.472 INFO:tasks.rados.rados.0.smithi033.stdout:354: delete oid 4 current snap is 30 2024-09-21T09:52:26.474 INFO:tasks.rados.rados.0.smithi033.stdout:354: done (0 left) 2024-09-21T09:52:26.475 INFO:tasks.rados.rados.0.smithi033.stdout:355: snap_create 2024-09-21T09:52:26.620 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:26 smithi105 podman[66951]: 2024-09-21 09:52:26.326948163 +0000 UTC m=+0.175447719 container died e1080a983fa6209fe2e4b292bbf5ad5b0bc8abea5497671886246e8b223ede9a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, RELEASE=quincy-fbd1a6a, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T09:52:26.620 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:26 smithi105 ceph-mon[31286]: pgmap v189: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1020 KiB/s wr, 14 op/s 2024-09-21T09:52:26.620 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:26 smithi105 ceph-mon[31286]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:52:26.620 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:26 smithi105 ceph-mon[31286]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:52:26.979 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:26 smithi105 podman[66951]: 2024-09-21 09:52:26.713502921 +0000 UTC m=+0.562002471 container cleanup e1080a983fa6209fe2e4b292bbf5ad5b0bc8abea5497671886246e8b223ede9a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:52:26.979 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:26 smithi105 bash[66951]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:52:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:26 smithi033 ceph-mon[39045]: pgmap v189: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1020 KiB/s wr, 14 op/s 2024-09-21T09:52:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:26 smithi033 ceph-mon[39045]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:52:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:26 smithi033 ceph-mon[39045]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:52:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:26 smithi033 ceph-mon[27969]: pgmap v189: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1020 KiB/s wr, 14 op/s 2024-09-21T09:52:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:26 smithi033 ceph-mon[27969]: osdmap e129: 8 total, 8 up, 8 in 2024-09-21T09:52:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:26 smithi033 ceph-mon[27969]: osdmap e130: 8 total, 8 up, 8 in 2024-09-21T09:52:27.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:26 smithi105 podman[66968]: 2024-09-21 09:52:26.977626418 +0000 UTC m=+0.646550392 container remove e1080a983fa6209fe2e4b292bbf5ad5b0bc8abea5497671886246e8b223ede9a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T09:52:27.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:26 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:52:27.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:52:27.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 systemd[1]: Stopped Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:52:27.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x.service: Consumed 10.245s CPU time. 2024-09-21T09:52:27.473 INFO:tasks.rados.rados.0.smithi033.stdout:355: done (0 left) 2024-09-21T09:52:27.473 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 31 2024-09-21T09:52:27.474 INFO:tasks.rados.rados.0.smithi033.stdout:356: seq_num 127 ranges {0=532480,532480=507904,1040384=778240,1818624=319488} 2024-09-21T09:52:27.476 INFO:tasks.rados.rados.0.smithi033.stdout:356: writing smithi03385152-28 from 0 to 532480 tid 1 2024-09-21T09:52:27.479 INFO:tasks.rados.rados.0.smithi033.stdout:356: writing smithi03385152-28 from 532480 to 1040384 tid 2 2024-09-21T09:52:27.483 INFO:tasks.rados.rados.0.smithi033.stdout:356: writing smithi03385152-28 from 1040384 to 1818624 tid 3 2024-09-21T09:52:27.485 INFO:tasks.rados.rados.0.smithi033.stdout:356: writing smithi03385152-28 from 1818624 to 2138112 tid 4 2024-09-21T09:52:27.486 INFO:tasks.rados.rados.0.smithi033.stdout:357: rollback oid 34 current snap is 31 2024-09-21T09:52:27.486 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 19 2024-09-21T09:52:27.486 INFO:tasks.rados.rados.0.smithi033.stdout:356: finishing write tid 1 to smithi03385152-28 2024-09-21T09:52:27.486 INFO:tasks.rados.rados.0.smithi033.stdout:356: finishing write tid 2 to smithi03385152-28 2024-09-21T09:52:27.486 INFO:tasks.rados.rados.0.smithi033.stdout:358: delete oid 8 current snap is 31 2024-09-21T09:52:27.488 INFO:tasks.rados.rados.0.smithi033.stdout:358: done (2 left) 2024-09-21T09:52:27.488 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 31 2024-09-21T09:52:27.488 INFO:tasks.rados.rados.0.smithi033.stdout:359: seq_num 129 ranges {778240=565248,1343488=679936,2023424=786432,2809856=139264} 2024-09-21T09:52:27.495 INFO:tasks.rados.rados.0.smithi033.stdout:359: writing smithi03385152-47 from 778240 to 1343488 tid 1 2024-09-21T09:52:27.499 INFO:tasks.rados.rados.0.smithi033.stdout:359: writing smithi03385152-47 from 1343488 to 2023424 tid 2 2024-09-21T09:52:27.504 INFO:tasks.rados.rados.0.smithi033.stdout:359: writing smithi03385152-47 from 2023424 to 2809856 tid 3 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:359: writing smithi03385152-47 from 2809856 to 2949120 tid 4 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:360: snap_create 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:356: finishing write tid 3 to smithi03385152-28 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:356: finishing write tid 4 to smithi03385152-28 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:357: finishing rollback tid 0 to smithi03385152-34 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:357: finishing rollback tid 1 to smithi03385152-34 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:357: finishing rollback tid 2 to smithi03385152-34 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 196 (ObjNum 88 snap 16 seq_num 88) dirty exists 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:356: finishing write tid 5 to smithi03385152-28 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:356: finishing write tid 6 to smithi03385152-28 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 47 (ObjNum 127 snap 31 seq_num 127) dirty exists 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:356: left oid 28 (ObjNum 127 snap 31 seq_num 127) 2024-09-21T09:52:27.505 INFO:tasks.rados.rados.0.smithi033.stdout:359: finishing write tid 1 to smithi03385152-47 2024-09-21T09:52:27.506 INFO:tasks.rados.rados.0.smithi033.stdout:359: finishing write tid 2 to smithi03385152-47 2024-09-21T09:52:27.507 INFO:tasks.rados.rados.0.smithi033.stdout:359: finishing write tid 3 to smithi03385152-47 2024-09-21T09:52:27.507 INFO:tasks.rados.rados.0.smithi033.stdout:359: finishing write tid 4 to smithi03385152-47 2024-09-21T09:52:27.507 INFO:tasks.rados.rados.0.smithi033.stdout:359: finishing write tid 5 to smithi03385152-47 2024-09-21T09:52:27.508 INFO:tasks.rados.rados.0.smithi033.stdout:359: finishing write tid 6 to smithi03385152-47 2024-09-21T09:52:27.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 8 (ObjNum 129 snap 31 seq_num 129) dirty exists 2024-09-21T09:52:27.508 INFO:tasks.rados.rados.0.smithi033.stdout:359: left oid 47 (ObjNum 129 snap 31 seq_num 129) 2024-09-21T09:52:27.724 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 systemd[1]: Starting Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:52:27.724 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 podman[67105]: 2024-09-21 09:52:27.471827048 +0000 UTC m=+0.016262405 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:52:27.724 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 podman[67105]: 2024-09-21 09:52:27.60375334 +0000 UTC m=+0.148188677 container create b10e03ec7c52c92d932372e2f7be47f83bc400a43de6182794c71ef4fb36cd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T09:52:28.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 podman[67105]: 2024-09-21 09:52:27.839863742 +0000 UTC m=+0.384299087 container init b10e03ec7c52c92d932372e2f7be47f83bc400a43de6182794c71ef4fb36cd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T09:52:28.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 podman[67105]: 2024-09-21 09:52:27.844533011 +0000 UTC m=+0.388968349 container start b10e03ec7c52c92d932372e2f7be47f83bc400a43de6182794c71ef4fb36cd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:52:28.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 ceph-mgr[67143]: -- 172.21.15.105:0/2620519991 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56472d90d0e0 con 0x56472d8eb400 2024-09-21T09:52:28.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 bash[67105]: b10e03ec7c52c92d932372e2f7be47f83bc400a43de6182794c71ef4fb36cd54 2024-09-21T09:52:28.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:27 smithi105 systemd[1]: Started Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:356: done (3 left) 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:357: done (2 left) 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:359: done (1 left) 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:360: done (0 left) 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:361: read oid 45 snap -1 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:361: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:362: rmattr oid 18 current snap is 32 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:362: done (1 left) 2024-09-21T09:52:28.482 INFO:tasks.rados.rados.0.smithi033.stdout:363: read oid 28 snap -1 2024-09-21T09:52:28.483 INFO:tasks.rados.rados.0.smithi033.stdout:363: expect (ObjNum 127 snap 31 seq_num 127) 2024-09-21T09:52:28.483 INFO:tasks.rados.rados.0.smithi033.stdout:364: copy_from oid 19 from oid 20 current snap is 32 2024-09-21T09:52:28.483 INFO:tasks.rados.rados.0.smithi033.stdout:365: rmattr oid 26 current snap is 32 2024-09-21T09:52:28.483 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 32 2024-09-21T09:52:28.483 INFO:tasks.rados.rados.0.smithi033.stdout:366: seq_num 130 ranges {1687552=581632,2269184=614400,2883584=598016,3481600=393216} 2024-09-21T09:52:28.494 INFO:tasks.rados.rados.0.smithi033.stdout:366: writing smithi03385152-23 from 1687552 to 2269184 tid 1 2024-09-21T09:52:28.498 INFO:tasks.rados.rados.0.smithi033.stdout:366: writing smithi03385152-23 from 2269184 to 2883584 tid 2 2024-09-21T09:52:28.501 INFO:tasks.rados.rados.0.smithi033.stdout:366: writing smithi03385152-23 from 2883584 to 3481600 tid 3 2024-09-21T09:52:28.504 INFO:tasks.rados.rados.0.smithi033.stdout:366: writing smithi03385152-23 from 3481600 to 3874816 tid 4 2024-09-21T09:52:28.504 INFO:tasks.rados.rados.0.smithi033.stdout:367: delete oid 42 current snap is 32 2024-09-21T09:52:28.504 INFO:tasks.rados.rados.0.smithi033.stdout:364: finishing copy_from to smithi03385152-19 2024-09-21T09:52:28.504 INFO:tasks.rados.rados.0.smithi033.stdout:364: got expected ENOENT (src dne) 2024-09-21T09:52:28.504 INFO:tasks.rados.rados.0.smithi033.stdout:364: finishing copy_from racing read to smithi03385152-19 2024-09-21T09:52:28.504 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 107 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:52:28.505 INFO:tasks.rados.rados.0.smithi033.stdout:361: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:52:28.513 INFO:tasks.rados.rados.0.smithi033.stdout:363: expect (ObjNum 127 snap 31 seq_num 127) 2024-09-21T09:52:28.569 INFO:tasks.rados.rados.0.smithi033.stdout:366: finishing write tid 1 to smithi03385152-23 2024-09-21T09:52:28.569 INFO:tasks.rados.rados.0.smithi033.stdout:366: finishing write tid 2 to smithi03385152-23 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:366: finishing write tid 3 to smithi03385152-23 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:361: done (5 left) 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:363: done (4 left) 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:364: done (3 left) 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:365: done (2 left) 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:367: done (1 left) 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:368: delete oid 4 current snap is 32 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:366: finishing write tid 4 to smithi03385152-23 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:366: finishing write tid 5 to smithi03385152-23 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:366: finishing write tid 6 to smithi03385152-23 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 136 (ObjNum 130 snap 32 seq_num 130) dirty exists 2024-09-21T09:52:28.570 INFO:tasks.rados.rados.0.smithi033.stdout:366: left oid 23 (ObjNum 130 snap 32 seq_num 130) 2024-09-21T09:52:28.571 INFO:tasks.rados.rados.0.smithi033.stdout:366: done (1 left) 2024-09-21T09:52:28.571 INFO:tasks.rados.rados.0.smithi033.stdout:368: done (0 left) 2024-09-21T09:52:28.571 INFO:tasks.rados.rados.0.smithi033.stdout:369: read oid 33 snap 23 2024-09-21T09:52:28.571 INFO:tasks.rados.rados.0.smithi033.stdout:369: expect (ObjNum 104 snap 20 seq_num 104) 2024-09-21T09:52:28.571 INFO:tasks.rados.rados.0.smithi033.stdout:370: setattr oid 15 current snap is 32 2024-09-21T09:52:28.572 INFO:tasks.rados.rados.0.smithi033.stdout:371: setattr oid 29 current snap is 32 2024-09-21T09:52:28.574 INFO:tasks.rados.rados.0.smithi033.stdout:372: delete oid 34 current snap is 32 2024-09-21T09:52:28.576 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 54 (ObjNum 75 snap 12 seq_num 75) dirty exists 2024-09-21T09:52:28.578 INFO:tasks.rados.rados.0.smithi033.stdout:370: done (3 left) 2024-09-21T09:52:28.578 INFO:tasks.rados.rados.0.smithi033.stdout:372: done (2 left) 2024-09-21T09:52:28.578 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 32 2024-09-21T09:52:28.578 INFO:tasks.rados.rados.0.smithi033.stdout:373: seq_num 134 ranges {2023424=647168,2670592=548864,3219456=573440,3792896=483328} 2024-09-21T09:52:28.592 INFO:tasks.rados.rados.0.smithi033.stdout:373: writing smithi03385152-36 from 2023424 to 2670592 tid 1 2024-09-21T09:52:28.596 INFO:tasks.rados.rados.0.smithi033.stdout:373: writing smithi03385152-36 from 2670592 to 3219456 tid 2 2024-09-21T09:52:28.599 INFO:tasks.rados.rados.0.smithi033.stdout:373: writing smithi03385152-36 from 3219456 to 3792896 tid 3 2024-09-21T09:52:28.602 INFO:tasks.rados.rados.0.smithi033.stdout:373: writing smithi03385152-36 from 3792896 to 4276224 tid 4 2024-09-21T09:52:28.602 INFO:tasks.rados.rados.0.smithi033.stdout:374: snap_remove snap 31 2024-09-21T09:52:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[39045]: pgmap v192: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1020 KiB/s wr, 14 op/s 2024-09-21T09:52:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[39045]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:52:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:28.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[27969]: pgmap v192: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1020 KiB/s wr, 14 op/s 2024-09-21T09:52:28.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[27969]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:52:28.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:28.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:28.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:28 smithi105 ceph-mon[31286]: pgmap v192: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 1020 KiB/s wr, 14 op/s 2024-09-21T09:52:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:28 smithi105 ceph-mon[31286]: osdmap e131: 8 total, 8 up, 8 in 2024-09-21T09:52:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:28 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:28 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:28 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:29.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:28] "GET /metrics HTTP/1.1" 200 240011 "" "Prometheus/2.43.0" 2024-09-21T09:52:29.498 INFO:tasks.rados.rados.0.smithi033.stdout:374: done (3 left) 2024-09-21T09:52:29.498 INFO:tasks.rados.rados.0.smithi033.stdout:375: delete oid 12 current snap is 32 2024-09-21T09:52:29.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 230 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:52:29.499 INFO:tasks.rados.rados.0.smithi033.stdout:369: expect (ObjNum 104 snap 20 seq_num 104) 2024-09-21T09:52:29.561 INFO:tasks.rados.rados.0.smithi033.stdout:373: finishing write tid 1 to smithi03385152-36 2024-09-21T09:52:29.561 INFO:tasks.rados.rados.0.smithi033.stdout:373: finishing write tid 2 to smithi03385152-36 2024-09-21T09:52:29.561 INFO:tasks.rados.rados.0.smithi033.stdout:373: finishing write tid 3 to smithi03385152-36 2024-09-21T09:52:29.561 INFO:tasks.rados.rados.0.smithi033.stdout:373: finishing write tid 4 to smithi03385152-36 2024-09-21T09:52:29.561 INFO:tasks.rados.rados.0.smithi033.stdout:373: finishing write tid 5 to smithi03385152-36 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:373: finishing write tid 6 to smithi03385152-36 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 236 (ObjNum 134 snap 32 seq_num 134) dirty exists 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:373: left oid 36 (ObjNum 134 snap 32 seq_num 134) 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:369: done (3 left) 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:371: done (2 left) 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:373: done (1 left) 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:375: done (0 left) 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:376: rollback oid 9 current snap is 32 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 9 to 22 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:377: read oid 11 snap 19 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:377: expect (ObjNum 101 snap 17 seq_num 101) 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 32 2024-09-21T09:52:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:378: seq_num 136 ranges {3874816=688128,4562944=720896,5283840=540672,5824512=344064} 2024-09-21T09:52:29.566 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:29 smithi105 ceph-mon[31286]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:52:29.566 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:29 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:378: writing smithi03385152-23 from 3874816 to 4562944 tid 1 2024-09-21T09:52:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:378: writing smithi03385152-23 from 4562944 to 5283840 tid 2 2024-09-21T09:52:29.594 INFO:tasks.rados.rados.0.smithi033.stdout:378: writing smithi03385152-23 from 5283840 to 5824512 tid 3 2024-09-21T09:52:29.596 INFO:tasks.rados.rados.0.smithi033.stdout:378: writing smithi03385152-23 from 5824512 to 6168576 tid 4 2024-09-21T09:52:29.596 INFO:tasks.rados.rados.0.smithi033.stdout:379: delete oid 7 current snap is 32 2024-09-21T09:52:29.596 INFO:tasks.rados.rados.0.smithi033.stdout:376: finishing rollback tid 0 to smithi03385152-9 2024-09-21T09:52:29.596 INFO:tasks.rados.rados.0.smithi033.stdout:376: finishing rollback tid 1 to smithi03385152-9 2024-09-21T09:52:29.596 INFO:tasks.rados.rados.0.smithi033.stdout:376: finishing rollback tid 2 to smithi03385152-9 2024-09-21T09:52:29.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 106 (ObjNum 69 snap 7 seq_num 69) dirty exists 2024-09-21T09:52:29.596 INFO:tasks.rados.rados.0.smithi033.stdout:377: expect (ObjNum 101 snap 17 seq_num 101) 2024-09-21T09:52:29.642 INFO:tasks.rados.rados.0.smithi033.stdout:378: finishing write tid 1 to smithi03385152-23 2024-09-21T09:52:29.642 INFO:tasks.rados.rados.0.smithi033.stdout:378: finishing write tid 2 to smithi03385152-23 2024-09-21T09:52:29.642 INFO:tasks.rados.rados.0.smithi033.stdout:378: finishing write tid 3 to smithi03385152-23 2024-09-21T09:52:29.642 INFO:tasks.rados.rados.0.smithi033.stdout:378: finishing write tid 4 to smithi03385152-23 2024-09-21T09:52:29.642 INFO:tasks.rados.rados.0.smithi033.stdout:378: finishing write tid 5 to smithi03385152-23 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:378: finishing write tid 6 to smithi03385152-23 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 143 (ObjNum 136 snap 32 seq_num 136) dirty exists 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:378: left oid 23 (ObjNum 136 snap 32 seq_num 136) 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:376: done (3 left) 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:377: done (2 left) 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:378: done (1 left) 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:379: done (0 left) 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:380: read oid 16 snap 22 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:380: expect (ObjNum 102 snap 19 seq_num 102) 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 32 2024-09-21T09:52:29.643 INFO:tasks.rados.rados.0.smithi033.stdout:381: seq_num 138 ranges {2949120=720896,3670016=540672,4210688=557056,4767744=507904} 2024-09-21T09:52:29.662 INFO:tasks.rados.rados.0.smithi033.stdout:381: writing smithi03385152-47 from 2949120 to 3670016 tid 1 2024-09-21T09:52:29.665 INFO:tasks.rados.rados.0.smithi033.stdout:381: writing smithi03385152-47 from 3670016 to 4210688 tid 2 2024-09-21T09:52:29.668 INFO:tasks.rados.rados.0.smithi033.stdout:381: writing smithi03385152-47 from 4210688 to 4767744 tid 3 2024-09-21T09:52:29.671 INFO:tasks.rados.rados.0.smithi033.stdout:381: writing smithi03385152-47 from 4767744 to 5275648 tid 4 2024-09-21T09:52:29.672 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 32 2024-09-21T09:52:29.672 INFO:tasks.rados.rados.0.smithi033.stdout:380: expect (ObjNum 102 snap 19 seq_num 102) 2024-09-21T09:52:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:29 smithi033 ceph-mon[39045]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:52:29.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:29 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:29.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:29 smithi033 ceph-mon[27969]: osdmap e132: 8 total, 8 up, 8 in 2024-09-21T09:52:29.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:29 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:29.798 INFO:tasks.rados.rados.0.smithi033.stdout:381: finishing write tid 1 to smithi03385152-47 2024-09-21T09:52:29.798 INFO:tasks.rados.rados.0.smithi033.stdout:381: finishing write tid 2 to smithi03385152-47 2024-09-21T09:52:29.798 INFO:tasks.rados.rados.0.smithi033.stdout:381: finishing write tid 3 to smithi03385152-47 2024-09-21T09:52:29.798 INFO:tasks.rados.rados.0.smithi033.stdout:381: finishing write tid 4 to smithi03385152-47 2024-09-21T09:52:29.798 INFO:tasks.rados.rados.0.smithi033.stdout:381: finishing write tid 5 to smithi03385152-47 2024-09-21T09:52:29.799 INFO:tasks.rados.rados.0.smithi033.stdout:381: finishing write tid 6 to smithi03385152-47 2024-09-21T09:52:29.799 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 13 (ObjNum 138 snap 32 seq_num 138) dirty exists 2024-09-21T09:52:29.799 INFO:tasks.rados.rados.0.smithi033.stdout:381: left oid 47 (ObjNum 138 snap 32 seq_num 138) 2024-09-21T09:52:29.799 INFO:tasks.rados.rados.0.smithi033.stdout:382: seq_num 139 ranges {1261568=737280,1998848=425984,2424832=761856,3186688=417792} 2024-09-21T09:52:29.809 INFO:tasks.rados.rados.0.smithi033.stdout:382: writing smithi03385152-44 from 1261568 to 1998848 tid 1 2024-09-21T09:52:29.811 INFO:tasks.rados.rados.0.smithi033.stdout:382: writing smithi03385152-44 from 1998848 to 2424832 tid 2 2024-09-21T09:52:29.815 INFO:tasks.rados.rados.0.smithi033.stdout:382: writing smithi03385152-44 from 2424832 to 3186688 tid 3 2024-09-21T09:52:29.818 INFO:tasks.rados.rados.0.smithi033.stdout:382: writing smithi03385152-44 from 3186688 to 3604480 tid 4 2024-09-21T09:52:29.818 INFO:tasks.rados.rados.0.smithi033.stdout:380: done (2 left) 2024-09-21T09:52:29.818 INFO:tasks.rados.rados.0.smithi033.stdout:381: done (1 left) 2024-09-21T09:52:29.818 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 32 2024-09-21T09:52:29.818 INFO:tasks.rados.rados.0.smithi033.stdout:382: finishing write tid 1 to smithi03385152-44 2024-09-21T09:52:29.819 INFO:tasks.rados.rados.0.smithi033.stdout:382: finishing write tid 2 to smithi03385152-44 2024-09-21T09:52:29.819 INFO:tasks.rados.rados.0.smithi033.stdout:383: seq_num 140 ranges {0=753664,753664=704512,1458176=524288,1982464=376832} 2024-09-21T09:52:29.822 INFO:tasks.rados.rados.0.smithi033.stdout:383: writing smithi03385152-1 from 0 to 753664 tid 1 2024-09-21T09:52:29.826 INFO:tasks.rados.rados.0.smithi033.stdout:383: writing smithi03385152-1 from 753664 to 1458176 tid 2 2024-09-21T09:52:29.829 INFO:tasks.rados.rados.0.smithi033.stdout:383: writing smithi03385152-1 from 1458176 to 1982464 tid 3 2024-09-21T09:52:29.831 INFO:tasks.rados.rados.0.smithi033.stdout:383: writing smithi03385152-1 from 1982464 to 2359296 tid 4 2024-09-21T09:52:29.832 INFO:tasks.rados.rados.0.smithi033.stdout:384: snap_remove snap 23 2024-09-21T09:52:30.494 INFO:tasks.rados.rados.0.smithi033.stdout:384: done (2 left) 2024-09-21T09:52:30.494 INFO:tasks.rados.rados.0.smithi033.stdout:385: rollback oid 43 current snap is 32 2024-09-21T09:52:30.494 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 43 to 29 2024-09-21T09:52:30.494 INFO:tasks.rados.rados.0.smithi033.stdout:382: finishing write tid 3 to smithi03385152-44 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:382: finishing write tid 4 to smithi03385152-44 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:382: finishing write tid 5 to smithi03385152-44 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:382: finishing write tid 6 to smithi03385152-44 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 50 (ObjNum 139 snap 32 seq_num 139) dirty exists 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:382: left oid 44 (ObjNum 139 snap 32 seq_num 139) 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:383: finishing write tid 1 to smithi03385152-1 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:383: finishing write tid 2 to smithi03385152-1 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:383: finishing write tid 3 to smithi03385152-1 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:383: finishing write tid 4 to smithi03385152-1 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:383: finishing write tid 5 to smithi03385152-1 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:383: finishing write tid 6 to smithi03385152-1 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 62 (ObjNum 140 snap 32 seq_num 140) dirty exists 2024-09-21T09:52:30.495 INFO:tasks.rados.rados.0.smithi033.stdout:383: left oid 1 (ObjNum 140 snap 32 seq_num 140) 2024-09-21T09:52:30.496 INFO:tasks.rados.rados.0.smithi033.stdout:382: done (2 left) 2024-09-21T09:52:30.496 INFO:tasks.rados.rados.0.smithi033.stdout:383: done (1 left) 2024-09-21T09:52:30.496 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 32 2024-09-21T09:52:30.496 INFO:tasks.rados.rados.0.smithi033.stdout:386: seq_num 141 ranges {3858432=770048,4628480=589824,5218304=729088,5947392=286720} 2024-09-21T09:52:30.521 INFO:tasks.rados.rados.0.smithi033.stdout:386: writing smithi03385152-14 from 3858432 to 4628480 tid 1 2024-09-21T09:52:30.524 INFO:tasks.rados.rados.0.smithi033.stdout:386: writing smithi03385152-14 from 4628480 to 5218304 tid 2 2024-09-21T09:52:30.528 INFO:tasks.rados.rados.0.smithi033.stdout:386: writing smithi03385152-14 from 5218304 to 5947392 tid 3 2024-09-21T09:52:30.530 INFO:tasks.rados.rados.0.smithi033.stdout:386: writing smithi03385152-14 from 5947392 to 6234112 tid 4 2024-09-21T09:52:30.530 INFO:tasks.rados.rados.0.smithi033.stdout:387: copy_from oid 6 from oid 19 current snap is 32 2024-09-21T09:52:30.530 INFO:tasks.rados.rados.0.smithi033.stdout:385: finishing rollback tid 0 to smithi03385152-43 2024-09-21T09:52:30.531 INFO:tasks.rados.rados.0.smithi033.stdout:385: finishing rollback tid 1 to smithi03385152-43 2024-09-21T09:52:30.531 INFO:tasks.rados.rados.0.smithi033.stdout:385: finishing rollback tid 2 to smithi03385152-43 2024-09-21T09:52:30.531 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 138 (ObjNum 72 snap 10 seq_num 72) dirty exists 2024-09-21T09:52:30.531 INFO:tasks.rados.rados.0.smithi033.stdout:386: finishing write tid 1 to smithi03385152-14 2024-09-21T09:52:30.531 INFO:tasks.rados.rados.0.smithi033.stdout:386: finishing write tid 2 to smithi03385152-14 2024-09-21T09:52:30.531 INFO:tasks.rados.rados.0.smithi033.stdout:385: done (2 left) 2024-09-21T09:52:30.531 INFO:tasks.rados.rados.0.smithi033.stdout:388: snap_create 2024-09-21T09:52:30.533 INFO:tasks.rados.rados.0.smithi033.stdout:386: finishing write tid 3 to smithi03385152-14 2024-09-21T09:52:30.533 INFO:tasks.rados.rados.0.smithi033.stdout:386: finishing write tid 4 to smithi03385152-14 2024-09-21T09:52:30.533 INFO:tasks.rados.rados.0.smithi033.stdout:386: finishing write tid 5 to smithi03385152-14 2024-09-21T09:52:30.534 INFO:tasks.rados.rados.0.smithi033.stdout:386: finishing write tid 6 to smithi03385152-14 2024-09-21T09:52:30.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 218 (ObjNum 141 snap 32 seq_num 141) dirty exists 2024-09-21T09:52:30.534 INFO:tasks.rados.rados.0.smithi033.stdout:386: left oid 14 (ObjNum 141 snap 32 seq_num 141) 2024-09-21T09:52:30.538 INFO:tasks.rados.rados.0.smithi033.stdout:387: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:52:30.539 INFO:tasks.rados.rados.0.smithi033.stdout:387: finishing copy_from to smithi03385152-6 2024-09-21T09:52:30.539 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 20 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:52:30.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:30 smithi105 ceph-mon[31286]: pgmap v195: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:30.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:30 smithi105 ceph-mon[31286]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:52:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:30 smithi033 ceph-mon[39045]: pgmap v195: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:30 smithi033 ceph-mon[39045]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:52:30.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:30 smithi033 ceph-mon[27969]: pgmap v195: 121 pgs: 121 active+clean; 94 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:30.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:30 smithi033 ceph-mon[27969]: osdmap e133: 8 total, 8 up, 8 in 2024-09-21T09:52:31.509 INFO:tasks.rados.rados.0.smithi033.stdout:386: done (2 left) 2024-09-21T09:52:31.509 INFO:tasks.rados.rados.0.smithi033.stdout:387: done (1 left) 2024-09-21T09:52:31.509 INFO:tasks.rados.rados.0.smithi033.stdout:388: done (0 left) 2024-09-21T09:52:31.509 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 33 2024-09-21T09:52:31.509 INFO:tasks.rados.rados.0.smithi033.stdout:389: seq_num 142 ranges {0=786432,786432=475136,1261568=540672,1802240=589824} 2024-09-21T09:52:31.513 INFO:tasks.rados.rados.0.smithi033.stdout:389: writing smithi03385152-12 from 0 to 786432 tid 1 2024-09-21T09:52:31.515 INFO:tasks.rados.rados.0.smithi033.stdout:389: writing smithi03385152-12 from 786432 to 1261568 tid 2 2024-09-21T09:52:31.518 INFO:tasks.rados.rados.0.smithi033.stdout:389: writing smithi03385152-12 from 1261568 to 1802240 tid 3 2024-09-21T09:52:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:389: writing smithi03385152-12 from 1802240 to 2392064 tid 4 2024-09-21T09:52:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:390: read oid 9 snap -1 2024-09-21T09:52:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:390: expect (ObjNum 69 snap 7 seq_num 69) 2024-09-21T09:52:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 33 2024-09-21T09:52:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:389: finishing write tid 1 to smithi03385152-12 2024-09-21T09:52:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:389: finishing write tid 2 to smithi03385152-12 2024-09-21T09:52:31.523 INFO:tasks.rados.rados.0.smithi033.stdout:389: finishing write tid 3 to smithi03385152-12 2024-09-21T09:52:31.523 INFO:tasks.rados.rados.0.smithi033.stdout:391: seq_num 143 ranges {0=802816,802816=761856,1564672=745472,2310144=98304} 2024-09-21T09:52:31.527 INFO:tasks.rados.rados.0.smithi033.stdout:391: writing smithi03385152-26 from 0 to 802816 tid 1 2024-09-21T09:52:31.531 INFO:tasks.rados.rados.0.smithi033.stdout:391: writing smithi03385152-26 from 802816 to 1564672 tid 2 2024-09-21T09:52:31.536 INFO:tasks.rados.rados.0.smithi033.stdout:391: writing smithi03385152-26 from 1564672 to 2310144 tid 3 2024-09-21T09:52:31.537 INFO:tasks.rados.rados.0.smithi033.stdout:391: writing smithi03385152-26 from 2310144 to 2408448 tid 4 2024-09-21T09:52:31.537 INFO:tasks.rados.rados.0.smithi033.stdout:392: snap_create 2024-09-21T09:52:31.537 INFO:tasks.rados.rados.0.smithi033.stdout:389: finishing write tid 4 to smithi03385152-12 2024-09-21T09:52:31.537 INFO:tasks.rados.rados.0.smithi033.stdout:389: finishing write tid 5 to smithi03385152-12 2024-09-21T09:52:31.537 INFO:tasks.rados.rados.0.smithi033.stdout:390: expect (ObjNum 69 snap 7 seq_num 69) 2024-09-21T09:52:31.605 INFO:tasks.rados.rados.0.smithi033.stdout:389: finishing write tid 6 to smithi03385152-12 2024-09-21T09:52:31.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 122 (ObjNum 142 snap 33 seq_num 142) dirty exists 2024-09-21T09:52:31.605 INFO:tasks.rados.rados.0.smithi033.stdout:389: left oid 12 (ObjNum 142 snap 33 seq_num 142) 2024-09-21T09:52:31.605 INFO:tasks.rados.rados.0.smithi033.stdout:391: finishing write tid 1 to smithi03385152-26 2024-09-21T09:52:31.605 INFO:tasks.rados.rados.0.smithi033.stdout:391: finishing write tid 2 to smithi03385152-26 2024-09-21T09:52:31.605 INFO:tasks.rados.rados.0.smithi033.stdout:391: finishing write tid 3 to smithi03385152-26 2024-09-21T09:52:31.606 INFO:tasks.rados.rados.0.smithi033.stdout:391: finishing write tid 4 to smithi03385152-26 2024-09-21T09:52:31.606 INFO:tasks.rados.rados.0.smithi033.stdout:391: finishing write tid 5 to smithi03385152-26 2024-09-21T09:52:31.606 INFO:tasks.rados.rados.0.smithi033.stdout:391: finishing write tid 6 to smithi03385152-26 2024-09-21T09:52:31.606 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 128 (ObjNum 143 snap 33 seq_num 143) dirty exists 2024-09-21T09:52:31.606 INFO:tasks.rados.rados.0.smithi033.stdout:391: left oid 26 (ObjNum 143 snap 33 seq_num 143) 2024-09-21T09:52:31.703 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:52:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:31 smithi105 ceph-mon[31286]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:52:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:31 smithi033 ceph-mon[39045]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:52:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:31 smithi033 ceph-mon[27969]: osdmap e134: 8 total, 8 up, 8 in 2024-09-21T09:52:32.231 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:52:32.232 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (2m) 2m ago 3m 17.3M - 0.25.0 c8568f914cd2 f5c871f25675 2024-09-21T09:52:32.232 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (3m) 2m ago 3m 82.5M - 9.4.7 954c08fa6188 39f7f9958856 2024-09-21T09:52:32.232 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (5m) 2m ago 5m 13.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:52:32.232 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (5m) 2m ago 5m 16.8M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:52:32.232 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283 starting - - - - 2024-09-21T09:52:32.232 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:9283,8765 running (10m) 2m ago 10m 445M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 21061eedc154 2024-09-21T09:52:32.232 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (10m) 2m ago 10m 48.2M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 220cb0bc67bb 2024-09-21T09:52:32.233 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (9m) 2m ago 9m 40.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c2741af3f853 2024-09-21T09:52:32.233 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (9m) 2m ago 9m 41.4M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f8b68f40cc5 2024-09-21T09:52:32.233 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (3m) 2m ago 3m 13.0M - 1.5.0 0da6a335fe13 ae258c372f47 2024-09-21T09:52:32.233 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (3m) 2m ago 3m 3376k - 1.5.0 0da6a335fe13 446328798f33 2024-09-21T09:52:32.233 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (8m) 2m ago 8m 43.9M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:52:32.233 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (8m) 2m ago 8m 44.3M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:52:32.233 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (7m) 2m ago 7m 44.8M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:52:32.234 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (7m) 2m ago 7m 39.8M 2993M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:52:32.234 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (6m) 2m ago 6m 42.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:52:32.234 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (6m) 2m ago 6m 42.6M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:52:32.234 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (6m) 2m ago 6m 42.5M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:52:32.234 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (5m) 2m ago 5m 42.4M 2992M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:52:32.234 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (2m) 2m ago 4m 24.0M - 2.43.0 a07b618ecd1d 9b3306609c60 2024-09-21T09:52:32.511 INFO:tasks.rados.rados.0.smithi033.stdout:389: done (3 left) 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:390: done (2 left) 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:391: done (1 left) 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:392: done (0 left) 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:393: copy_from oid 15 from oid 40 current snap is 34 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:394: read oid 7 snap 19 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:394: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:395: rollback oid 31 current snap is 34 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 31 to 33 2024-09-21T09:52:32.512 INFO:tasks.rados.rados.0.smithi033.stdout:396: delete oid 35 current snap is 34 2024-09-21T09:52:32.519 INFO:tasks.rados.rados.0.smithi033.stdout:395: finishing rollback tid 0 to smithi03385152-31 2024-09-21T09:52:32.519 INFO:tasks.rados.rados.0.smithi033.stdout:395: finishing rollback tid 1 to smithi03385152-31 2024-09-21T09:52:32.519 INFO:tasks.rados.rados.0.smithi033.stdout:395: finishing rollback tid 2 to smithi03385152-31 2024-09-21T09:52:32.519 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 252 (ObjNum 121 snap 28 seq_num 121) dirty exists 2024-09-21T09:52:32.519 INFO:tasks.rados.rados.0.smithi033.stdout:393: finishing copy_from racing read to smithi03385152-15 2024-09-21T09:52:32.520 INFO:tasks.rados.rados.0.smithi033.stdout:393: finishing copy_from to smithi03385152-15 2024-09-21T09:52:32.520 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 84 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:52:32.520 INFO:tasks.rados.rados.0.smithi033.stdout:394: expect (ObjNum 6 snap 0 seq_num 6) 2024-09-21T09:52:32.523 INFO:tasks.rados.rados.0.smithi033.stdout:393: done (3 left) 2024-09-21T09:52:32.523 INFO:tasks.rados.rados.0.smithi033.stdout:394: done (2 left) 2024-09-21T09:52:32.523 INFO:tasks.rados.rados.0.smithi033.stdout:395: done (1 left) 2024-09-21T09:52:32.523 INFO:tasks.rados.rados.0.smithi033.stdout:396: done (0 left) 2024-09-21T09:52:32.523 INFO:tasks.rados.rados.0.smithi033.stdout:397: read oid 41 snap -1 2024-09-21T09:52:32.523 INFO:tasks.rados.rados.0.smithi033.stdout:397: expect deleted 2024-09-21T09:52:32.524 INFO:tasks.rados.rados.0.smithi033.stdout:398: delete oid 23 current snap is 34 2024-09-21T09:52:32.526 INFO:tasks.rados.rados.0.smithi033.stdout:397: done (1 left) 2024-09-21T09:52:32.526 INFO:tasks.rados.rados.0.smithi033.stdout:398: done (0 left) 2024-09-21T09:52:32.526 INFO:tasks.rados.rados.0.smithi033.stdout:399: rmattr oid 9 current snap is 34 2024-09-21T09:52:32.526 INFO:tasks.rados.rados.0.smithi033.stdout:399: done (0 left) 2024-09-21T09:52:32.526 INFO:tasks.rados.rados.0.smithi033.stdout:400: copy_from oid 40 from oid 24 current snap is 34 2024-09-21T09:52:32.526 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 34 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:401: seq_num 146 ranges {0=49152} 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:401: writing smithi03385152-49 from 0 to 49152 tid 1 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:402: rollback oid 1 current snap is 34 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 29 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 34 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:403: seq_num 147 ranges {0=65536} 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:403: writing smithi03385152-35 from 0 to 65536 tid 1 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:404: copy_from oid 30 from oid 29 current snap is 34 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:405: read oid 25 snap -1 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:405: expect (ObjNum 99 snap 17 seq_num 99) 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:406: copy_from oid 44 from oid 7 current snap is 34 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:407: rmattr oid 41 current snap is 34 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:407: done (7 left) 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:408: rmattr oid 22 current snap is 34 2024-09-21T09:52:32.527 INFO:tasks.rados.rados.0.smithi033.stdout:408: done (7 left) 2024-09-21T09:52:32.528 INFO:tasks.rados.rados.0.smithi033.stdout:409: setattr oid 34 current snap is 34 2024-09-21T09:52:32.530 INFO:tasks.rados.rados.0.smithi033.stdout:401: finishing write tid 1 to smithi03385152-49 2024-09-21T09:52:32.530 INFO:tasks.rados.rados.0.smithi033.stdout:402: finishing rollback tid 0 to smithi03385152-1 2024-09-21T09:52:32.530 INFO:tasks.rados.rados.0.smithi033.stdout:410: rollback oid 38 current snap is 34 2024-09-21T09:52:32.530 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 30 2024-09-21T09:52:32.530 INFO:tasks.rados.rados.0.smithi033.stdout:411: delete oid 43 current snap is 34 2024-09-21T09:52:32.530 INFO:tasks.rados.rados.0.smithi033.stdout:401: finishing write tid 2 to smithi03385152-49 2024-09-21T09:52:32.531 INFO:tasks.rados.rados.0.smithi033.stdout:403: finishing write tid 1 to smithi03385152-35 2024-09-21T09:52:32.532 INFO:tasks.rados.rados.0.smithi033.stdout:401: finishing write tid 3 to smithi03385152-49 2024-09-21T09:52:32.532 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 130 (ObjNum 146 snap 34 seq_num 146) dirty exists 2024-09-21T09:52:32.532 INFO:tasks.rados.rados.0.smithi033.stdout:401: left oid 49 (ObjNum 146 snap 34 seq_num 146) 2024-09-21T09:52:32.533 INFO:tasks.rados.rados.0.smithi033.stdout:402: finishing rollback tid 1 to smithi03385152-1 2024-09-21T09:52:32.533 INFO:tasks.rados.rados.0.smithi033.stdout:402: finishing rollback tid 2 to smithi03385152-1 2024-09-21T09:52:32.533 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 87 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-21T09:52:32.533 INFO:tasks.rados.rados.0.smithi033.stdout:406: finishing copy_from to smithi03385152-44 2024-09-21T09:52:32.533 INFO:tasks.rados.rados.0.smithi033.stdout:406: got expected ENOENT (src dne) 2024-09-21T09:52:32.533 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 219 (ObjNum 945 snap 0 seq_num 91241840) dirty exists 2024-09-21T09:52:32.533 INFO:tasks.rados.rados.0.smithi033.stdout:406: finishing copy_from racing read to smithi03385152-44 2024-09-21T09:52:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:401: done (9 left) 2024-09-21T09:52:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:402: done (8 left) 2024-09-21T09:52:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:406: done (7 left) 2024-09-21T09:52:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:409: done (6 left) 2024-09-21T09:52:32.535 INFO:tasks.rados.rados.0.smithi033.stdout:411: done (5 left) 2024-09-21T09:52:32.535 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 34 2024-09-21T09:52:32.535 INFO:tasks.rados.rados.0.smithi033.stdout:412: seq_num 149 ranges {0=98304} 2024-09-21T09:52:32.536 INFO:tasks.rados.rados.0.smithi033.stdout:412: writing smithi03385152-5 from 0 to 98304 tid 1 2024-09-21T09:52:32.536 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 34 2024-09-21T09:52:32.536 INFO:tasks.rados.rados.0.smithi033.stdout:403: finishing write tid 2 to smithi03385152-35 2024-09-21T09:52:32.536 INFO:tasks.rados.rados.0.smithi033.stdout:413: seq_num 150 ranges {1638400=114688} 2024-09-21T09:52:32.545 INFO:tasks.rados.rados.0.smithi033.stdout:413: writing smithi03385152-13 from 1638400 to 1753088 tid 1 2024-09-21T09:52:32.546 INFO:tasks.rados.rados.0.smithi033.stdout:414: read oid 44 snap -1 2024-09-21T09:52:32.546 INFO:tasks.rados.rados.0.smithi033.stdout:414: expect (ObjNum 139 snap 32 seq_num 139) 2024-09-21T09:52:32.546 INFO:tasks.rados.rados.0.smithi033.stdout:415: read oid 15 snap -1 2024-09-21T09:52:32.546 INFO:tasks.rados.rados.0.smithi033.stdout:415: expect (ObjNum 39 snap 0 seq_num 39) 2024-09-21T09:52:32.546 INFO:tasks.rados.rados.0.smithi033.stdout:416: copy_from oid 23 from oid 27 current snap is 34 2024-09-21T09:52:32.546 INFO:tasks.rados.rados.0.smithi033.stdout:405: expect (ObjNum 99 snap 17 seq_num 99) 2024-09-21T09:52:32.571 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:52:32.571 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:52:32.571 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:52:32.571 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:52:32.571 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 15 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:52:32.572 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:400: finishing copy_from racing read to smithi03385152-40 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:410: finishing rollback tid 0 to smithi03385152-38 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:410: finishing rollback tid 1 to smithi03385152-38 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:405: done (9 left) 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:417: rollback oid 49 current snap is 34 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 49 to 19 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:412: finishing write tid 1 to smithi03385152-5 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:412: finishing write tid 2 to smithi03385152-5 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:400: finishing copy_from to smithi03385152-40 2024-09-21T09:52:32.592 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 16 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:410: finishing rollback tid 2 to smithi03385152-38 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 118 (ObjNum 116 snap 26 seq_num 116) dirty exists 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:412: finishing write tid 3 to smithi03385152-5 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 42 (ObjNum 149 snap 34 seq_num 149) dirty exists 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:412: left oid 5 (ObjNum 149 snap 34 seq_num 149) 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:404: finishing copy_from racing read to smithi03385152-30 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:403: finishing write tid 3 to smithi03385152-35 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 256 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:403: left oid 35 (ObjNum 147 snap 34 seq_num 147) 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:404: finishing copy_from to smithi03385152-30 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 230 (ObjNum 53 snap 1 seq_num 53) dirty exists 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:413: finishing write tid 1 to smithi03385152-13 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:413: finishing write tid 2 to smithi03385152-13 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:413: finishing write tid 3 to smithi03385152-13 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 261 (ObjNum 150 snap 34 seq_num 150) dirty exists 2024-09-21T09:52:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:413: left oid 13 (ObjNum 150 snap 34 seq_num 150) 2024-09-21T09:52:32.594 INFO:tasks.rados.rados.0.smithi033.stdout:415: expect (ObjNum 39 snap 0 seq_num 39) 2024-09-21T09:52:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:416: finishing copy_from racing read to smithi03385152-23 2024-09-21T09:52:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:416: finishing copy_from to smithi03385152-23 2024-09-21T09:52:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 46 (ObjNum 68 snap 6 seq_num 68) dirty exists 2024-09-21T09:52:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:414: expect (ObjNum 139 snap 32 seq_num 139) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:417: finishing rollback tid 0 to smithi03385152-49 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:417: finishing rollback tid 1 to smithi03385152-49 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:417: finishing rollback tid 2 to smithi03385152-49 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 133 (ObjNum 98 snap 17 seq_num 98) dirty exists 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:400: done (9 left) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:403: done (8 left) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:404: done (7 left) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:410: done (6 left) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:412: done (5 left) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:413: done (4 left) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:414: done (3 left) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:415: done (2 left) 2024-09-21T09:52:32.731 INFO:tasks.rados.rados.0.smithi033.stdout:416: done (1 left) 2024-09-21T09:52:32.732 INFO:tasks.rados.rados.0.smithi033.stdout:417: done (0 left) 2024-09-21T09:52:32.732 INFO:tasks.rados.rados.0.smithi033.stdout:418: delete oid 17 current snap is 34 2024-09-21T09:52:32.732 INFO:tasks.rados.rados.0.smithi033.stdout:418: done (0 left) 2024-09-21T09:52:32.732 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 34 2024-09-21T09:52:32.732 INFO:tasks.rados.rados.0.smithi033.stdout:419: seq_num 152 ranges {2138112=147456} 2024-09-21T09:52:32.745 INFO:tasks.rados.rados.0.smithi033.stdout:419: writing smithi03385152-28 from 2138112 to 2285568 tid 1 2024-09-21T09:52:32.745 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 34 2024-09-21T09:52:32.745 INFO:tasks.rados.rados.0.smithi033.stdout:420: seq_num 153 ranges {0=163840} 2024-09-21T09:52:32.746 INFO:tasks.rados.rados.0.smithi033.stdout:420: writing smithi03385152-17 from 0 to 163840 tid 1 2024-09-21T09:52:32.746 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 34 2024-09-21T09:52:32.746 INFO:tasks.rados.rados.0.smithi033.stdout:421: seq_num 154 ranges {2392064=180224} 2024-09-21T09:52:32.761 INFO:tasks.rados.rados.0.smithi033.stdout:421: writing smithi03385152-12 from 2392064 to 2572288 tid 1 2024-09-21T09:52:32.761 INFO:tasks.rados.rados.0.smithi033.stdout:422: read oid 34 snap -1 2024-09-21T09:52:32.761 INFO:tasks.rados.rados.0.smithi033.stdout:422: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:52:32.761 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 34 2024-09-21T09:52:32.761 INFO:tasks.rados.rados.0.smithi033.stdout:419: finishing write tid 1 to smithi03385152-28 2024-09-21T09:52:32.761 INFO:tasks.rados.rados.0.smithi033.stdout:423: seq_num 155 ranges {3080192=204800} 2024-09-21T09:52:32.779 INFO:tasks.rados.rados.0.smithi033.stdout:423: writing smithi03385152-38 from 3080192 to 3284992 tid 1 2024-09-21T09:52:32.779 INFO:tasks.rados.rados.0.smithi033.stdout:424: read oid 7 snap 33 2024-09-21T09:52:32.779 INFO:tasks.rados.rados.0.smithi033.stdout:424: expect deleted 2024-09-21T09:52:32.779 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 34 2024-09-21T09:52:32.779 INFO:tasks.rados.rados.0.smithi033.stdout:419: finishing write tid 2 to smithi03385152-28 2024-09-21T09:52:32.779 INFO:tasks.rados.rados.0.smithi033.stdout:420: finishing write tid 1 to smithi03385152-17 2024-09-21T09:52:32.780 INFO:tasks.rados.rados.0.smithi033.stdout:420: finishing write tid 2 to smithi03385152-17 2024-09-21T09:52:32.780 INFO:tasks.rados.rados.0.smithi033.stdout:419: finishing write tid 3 to smithi03385152-28 2024-09-21T09:52:32.780 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 52 (ObjNum 152 snap 34 seq_num 152) dirty exists 2024-09-21T09:52:32.780 INFO:tasks.rados.rados.0.smithi033.stdout:419: left oid 28 (ObjNum 152 snap 34 seq_num 152) 2024-09-21T09:52:32.780 INFO:tasks.rados.rados.0.smithi033.stdout:420: finishing write tid 3 to smithi03385152-17 2024-09-21T09:52:32.780 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 137 (ObjNum 153 snap 34 seq_num 153) dirty exists 2024-09-21T09:52:32.780 INFO:tasks.rados.rados.0.smithi033.stdout:420: left oid 17 (ObjNum 153 snap 34 seq_num 153) 2024-09-21T09:52:32.780 INFO:tasks.rados.rados.0.smithi033.stdout:425: seq_num 156 ranges {1720320=221184} 2024-09-21T09:52:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:32 smithi105 ceph-mon[31286]: pgmap v198: 121 pgs: 121 active+clean; 112 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2024-09-21T09:52:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:32 smithi105 ceph-mon[31286]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:52:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:32 smithi105 ceph-mon[31286]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:32.789 INFO:tasks.rados.rados.0.smithi033.stdout:425: writing smithi03385152-23 from 1720320 to 1941504 tid 1 2024-09-21T09:52:32.789 INFO:tasks.rados.rados.0.smithi033.stdout:419: done (6 left) 2024-09-21T09:52:32.789 INFO:tasks.rados.rados.0.smithi033.stdout:420: done (5 left) 2024-09-21T09:52:32.790 INFO:tasks.rados.rados.0.smithi033.stdout:426: delete oid 9 current snap is 34 2024-09-21T09:52:32.790 INFO:tasks.rados.rados.0.smithi033.stdout:422: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:52:32.797 INFO:tasks.rados.rados.0.smithi033.stdout:421: finishing write tid 1 to smithi03385152-12 2024-09-21T09:52:32.797 INFO:tasks.rados.rados.0.smithi033.stdout:421: finishing write tid 2 to smithi03385152-12 2024-09-21T09:52:32.797 INFO:tasks.rados.rados.0.smithi033.stdout:421: finishing write tid 3 to smithi03385152-12 2024-09-21T09:52:32.797 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 140 (ObjNum 154 snap 34 seq_num 154) dirty exists 2024-09-21T09:52:32.797 INFO:tasks.rados.rados.0.smithi033.stdout:421: left oid 12 (ObjNum 154 snap 34 seq_num 154) 2024-09-21T09:52:32.797 INFO:tasks.rados.rados.0.smithi033.stdout:421: done (5 left) 2024-09-21T09:52:32.797 INFO:tasks.rados.rados.0.smithi033.stdout:422: done (4 left) 2024-09-21T09:52:32.797 INFO:tasks.rados.rados.0.smithi033.stdout:426: done (3 left) 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:427: copy_from oid 18 from oid 47 current snap is 34 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:423: finishing write tid 1 to smithi03385152-38 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:423: finishing write tid 2 to smithi03385152-38 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:423: finishing write tid 3 to smithi03385152-38 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 120 (ObjNum 155 snap 34 seq_num 155) dirty exists 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:423: left oid 38 (ObjNum 155 snap 34 seq_num 155) 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:425: finishing write tid 1 to smithi03385152-23 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:425: finishing write tid 2 to smithi03385152-23 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:425: finishing write tid 3 to smithi03385152-23 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 154 (ObjNum 156 snap 34 seq_num 156) dirty exists 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:425: left oid 23 (ObjNum 156 snap 34 seq_num 156) 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:423: done (3 left) 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:424: done (2 left) 2024-09-21T09:52:32.798 INFO:tasks.rados.rados.0.smithi033.stdout:425: done (1 left) 2024-09-21T09:52:32.799 INFO:tasks.rados.rados.0.smithi033.stdout:428: read oid 35 snap 22 2024-09-21T09:52:32.799 INFO:tasks.rados.rados.0.smithi033.stdout:428: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:52:32.799 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 34 2024-09-21T09:52:32.799 INFO:tasks.rados.rados.0.smithi033.stdout:429: seq_num 158 ranges {5128192=253952} 2024-09-21T09:52:32.827 INFO:tasks.rados.rados.0.smithi033.stdout:429: writing smithi03385152-16 from 5128192 to 5382144 tid 1 2024-09-21T09:52:32.827 INFO:tasks.rados.rados.0.smithi033.stdout:430: delete oid 19 current snap is 34 2024-09-21T09:52:32.827 INFO:tasks.rados.rados.0.smithi033.stdout:428: expect (ObjNum 68 snap 6 seq_num 68) 2024-09-21T09:52:32.856 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:52:32.856 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:52:32.856 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:52:32.856 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:52:32.856 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-21T09:52:32.856 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "1/20 daemons upgraded", 2024-09-21T09:52:32.856 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T09:52:32.857 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:52:32.857 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:52:32.857 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:32 smithi033 ceph-mon[39045]: pgmap v198: 121 pgs: 121 active+clean; 112 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2024-09-21T09:52:32.857 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:32 smithi033 ceph-mon[39045]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:52:32.857 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:32 smithi033 ceph-mon[39045]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:32.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:32 smithi033 ceph-mon[27969]: pgmap v198: 121 pgs: 121 active+clean; 112 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2024-09-21T09:52:32.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:32 smithi033 ceph-mon[27969]: osdmap e135: 8 total, 8 up, 8 in 2024-09-21T09:52:32.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:32 smithi033 ceph-mon[27969]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:32.885 INFO:tasks.rados.rados.0.smithi033.stdout:427: finishing copy_from racing read to smithi03385152-18 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:427: finishing copy_from to smithi03385152-18 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 13 (ObjNum 138 snap 32 seq_num 138) dirty exists 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:429: finishing write tid 1 to smithi03385152-16 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:427: done (3 left) 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:428: done (2 left) 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:430: done (1 left) 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:431: snap_create 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:429: finishing write tid 2 to smithi03385152-16 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:429: finishing write tid 3 to smithi03385152-16 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 264 (ObjNum 158 snap 34 seq_num 158) dirty exists 2024-09-21T09:52:32.886 INFO:tasks.rados.rados.0.smithi033.stdout:429: left oid 16 (ObjNum 158 snap 34 seq_num 158) 2024-09-21T09:52:33.509 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:33] ENGINE Bus STARTING 2024-09-21T09:52:33.509 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: CherryPy Checker: 2024-09-21T09:52:33.509 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: The Application mounted at '' has an empty config. 2024-09-21T09:52:33.510 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:33.510 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:33] ENGINE Serving on http://:::9283 2024-09-21T09:52:33.510 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:33] ENGINE Bus STARTED 2024-09-21T09:52:33.778 INFO:tasks.rados.rados.0.smithi033.stdout:429: done (1 left) 2024-09-21T09:52:33.778 INFO:tasks.rados.rados.0.smithi033.stdout:431: done (0 left) 2024-09-21T09:52:33.778 INFO:tasks.rados.rados.0.smithi033.stdout:432: snap_create 2024-09-21T09:52:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='client.24926 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:52:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/3341074004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: Standby manager daemon x restarted 2024-09-21T09:52:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: Standby manager daemon x started 2024-09-21T09:52:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:52:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:52:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:33 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='client.24926 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:52:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/3341074004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: Standby manager daemon x restarted 2024-09-21T09:52:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: Standby manager daemon x started 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='client.24926 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='client.24932 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: osdmap e136: 8 total, 8 up, 8 in 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/3341074004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: Standby manager daemon x restarted 2024-09-21T09:52:33.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: Standby manager daemon x started 2024-09-21T09:52:33.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:52:33.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:33.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:52:33.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:33.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:33.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:33 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:34.773 INFO:tasks.rados.rados.0.smithi033.stdout:432: done (0 left) 2024-09-21T09:52:34.773 INFO:tasks.rados.rados.0.smithi033.stdout:433: snap_remove snap 34 2024-09-21T09:52:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:34 smithi105 ceph-mon[31286]: from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:34 smithi105 ceph-mon[31286]: pgmap v201: 121 pgs: 121 active+clean; 112 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2024-09-21T09:52:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:34 smithi105 ceph-mon[31286]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-21T09:52:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:34 smithi105 ceph-mon[31286]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:52:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:34 smithi033 ceph-mon[39045]: from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:34 smithi033 ceph-mon[39045]: pgmap v201: 121 pgs: 121 active+clean; 112 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2024-09-21T09:52:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:34 smithi033 ceph-mon[39045]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-21T09:52:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:34 smithi033 ceph-mon[39045]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:52:35.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:34 smithi033 ceph-mon[27969]: from='client.15042 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:52:35.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:34 smithi033 ceph-mon[27969]: pgmap v201: 121 pgs: 121 active+clean; 112 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.5 MiB/s rd, 4.7 MiB/s wr, 38 op/s 2024-09-21T09:52:35.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:34 smithi033 ceph-mon[27969]: mgrmap e23: y(active, since 4m), standbys: x 2024-09-21T09:52:35.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:34 smithi033 ceph-mon[27969]: osdmap e137: 8 total, 8 up, 8 in 2024-09-21T09:52:35.785 INFO:tasks.rados.rados.0.smithi033.stdout:433: done (0 left) 2024-09-21T09:52:35.785 INFO:tasks.rados.rados.0.smithi033.stdout:434: read oid 21 snap -1 2024-09-21T09:52:35.785 INFO:tasks.rados.rados.0.smithi033.stdout:434: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:52:35.785 INFO:tasks.rados.rados.0.smithi033.stdout:435: snap_remove snap 33 2024-09-21T09:52:36.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:35 smithi033 ceph-mon[39045]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:52:36.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:35 smithi033 ceph-mon[39045]: pgmap v204: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-21T09:52:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:35 smithi033 ceph-mon[27969]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:52:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:35 smithi033 ceph-mon[27969]: pgmap v204: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-21T09:52:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:35 smithi105 ceph-mon[31286]: osdmap e138: 8 total, 8 up, 8 in 2024-09-21T09:52:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:35 smithi105 ceph-mon[31286]: pgmap v204: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-21T09:52:36.475 INFO:tasks.rados.rados.0.smithi033.stdout:435: done (1 left) 2024-09-21T09:52:36.475 INFO:tasks.rados.rados.0.smithi033.stdout:436: snap_remove snap 32 2024-09-21T09:52:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:36 smithi105 ceph-mon[31286]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:52:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:36 smithi105 ceph-mon[31286]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:52:37.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:36 smithi033 ceph-mon[39045]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:52:37.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:36 smithi033 ceph-mon[39045]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:52:37.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:36 smithi033 ceph-mon[27969]: osdmap e139: 8 total, 8 up, 8 in 2024-09-21T09:52:37.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:36 smithi033 ceph-mon[27969]: osdmap e140: 8 total, 8 up, 8 in 2024-09-21T09:52:37.479 INFO:tasks.rados.rados.0.smithi033.stdout:436: done (1 left) 2024-09-21T09:52:37.479 INFO:tasks.rados.rados.0.smithi033.stdout:437: copy_from oid 40 from oid 22 current snap is 36 2024-09-21T09:52:37.480 INFO:tasks.rados.rados.0.smithi033.stdout:438: rollback oid 41 current snap is 36 2024-09-21T09:52:37.480 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 30 2024-09-21T09:52:37.480 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 36 2024-09-21T09:52:37.480 INFO:tasks.rados.rados.0.smithi033.stdout:439: seq_num 160 ranges {663552=286720} 2024-09-21T09:52:37.486 INFO:tasks.rados.rados.0.smithi033.stdout:439: writing smithi03385152-15 from 663552 to 950272 tid 1 2024-09-21T09:52:37.486 INFO:tasks.rados.rados.0.smithi033.stdout:440: snap_create 2024-09-21T09:52:37.486 INFO:tasks.rados.rados.0.smithi033.stdout:434: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:52:37.620 INFO:tasks.rados.rados.0.smithi033.stdout:438: finishing rollback tid 1 to smithi03385152-41 2024-09-21T09:52:37.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 0 (ObjNum 33 snap 0 seq_num 91102320) dirty dne 2024-09-21T09:52:37.620 INFO:tasks.rados.rados.0.smithi033.stdout:439: finishing write tid 1 to smithi03385152-15 2024-09-21T09:52:37.620 INFO:tasks.rados.rados.0.smithi033.stdout:439: finishing write tid 2 to smithi03385152-15 2024-09-21T09:52:37.620 INFO:tasks.rados.rados.0.smithi033.stdout:439: finishing write tid 3 to smithi03385152-15 2024-09-21T09:52:37.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 89 (ObjNum 160 snap 36 seq_num 160) dirty exists 2024-09-21T09:52:37.621 INFO:tasks.rados.rados.0.smithi033.stdout:439: left oid 15 (ObjNum 160 snap 36 seq_num 160) 2024-09-21T09:52:37.621 INFO:tasks.rados.rados.0.smithi033.stdout:437: finishing copy_from racing read to smithi03385152-40 2024-09-21T09:52:37.621 INFO:tasks.rados.rados.0.smithi033.stdout:437: finishing copy_from to smithi03385152-40 2024-09-21T09:52:37.621 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 140 (ObjNum 114 snap 25 seq_num 114) dirty exists 2024-09-21T09:52:38.481 INFO:tasks.rados.rados.0.smithi033.stdout:434: done (4 left) 2024-09-21T09:52:38.481 INFO:tasks.rados.rados.0.smithi033.stdout:437: done (3 left) 2024-09-21T09:52:38.481 INFO:tasks.rados.rados.0.smithi033.stdout:438: done (2 left) 2024-09-21T09:52:38.481 INFO:tasks.rados.rados.0.smithi033.stdout:439: done (1 left) 2024-09-21T09:52:38.481 INFO:tasks.rados.rados.0.smithi033.stdout:440: done (0 left) 2024-09-21T09:52:38.481 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 37 2024-09-21T09:52:38.481 INFO:tasks.rados.rados.0.smithi033.stdout:441: seq_num 161 ranges {163840=303104} 2024-09-21T09:52:38.484 INFO:tasks.rados.rados.0.smithi033.stdout:441: writing smithi03385152-17 from 163840 to 466944 tid 1 2024-09-21T09:52:38.484 INFO:tasks.rados.rados.0.smithi033.stdout:442: copy_from oid 28 from oid 42 current snap is 37 2024-09-21T09:52:38.484 INFO:tasks.rados.rados.0.smithi033.stdout:443: rollback oid 15 current snap is 37 2024-09-21T09:52:38.484 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 15 to 35 2024-09-21T09:52:38.484 INFO:tasks.rados.rados.0.smithi033.stdout:444: rollback oid 33 current snap is 37 2024-09-21T09:52:38.484 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 36 2024-09-21T09:52:38.485 INFO:tasks.rados.rados.0.smithi033.stdout:445: delete oid 46 current snap is 37 2024-09-21T09:52:38.485 INFO:tasks.rados.rados.0.smithi033.stdout:442: finishing copy_from to smithi03385152-28 2024-09-21T09:52:38.485 INFO:tasks.rados.rados.0.smithi033.stdout:442: got expected ENOENT (src dne) 2024-09-21T09:52:38.485 INFO:tasks.rados.rados.0.smithi033.stdout:442: finishing copy_from racing read to smithi03385152-28 2024-09-21T09:52:38.486 INFO:tasks.rados.rados.0.smithi033.stdout:443: finishing rollback tid 0 to smithi03385152-15 2024-09-21T09:52:38.492 INFO:tasks.rados.rados.0.smithi033.stdout:443: finishing rollback tid 1 to smithi03385152-15 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:442: done (4 left) 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:445: done (3 left) 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:446: read oid 22 snap 19 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:446: expect deleted 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:447: read oid 4 snap -1 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:447: expect deleted 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:448: copy_from oid 46 from oid 2 current snap is 37 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:449: delete oid 36 current snap is 37 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:441: finishing write tid 1 to smithi03385152-17 2024-09-21T09:52:38.493 INFO:tasks.rados.rados.0.smithi033.stdout:444: finishing rollback tid 0 to smithi03385152-33 2024-09-21T09:52:38.494 INFO:tasks.rados.rados.0.smithi033.stdout:441: finishing write tid 2 to smithi03385152-17 2024-09-21T09:52:38.494 INFO:tasks.rados.rados.0.smithi033.stdout:443: finishing rollback tid 2 to smithi03385152-15 2024-09-21T09:52:38.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 92 (ObjNum 39 snap 0 seq_num 39) dirty exists 2024-09-21T09:52:38.494 INFO:tasks.rados.rados.0.smithi033.stdout:444: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:52:38.494 INFO:tasks.rados.rados.0.smithi033.stdout:444: finishing rollback tid 2 to smithi03385152-33 2024-09-21T09:52:38.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 112 (ObjNum 104 snap 20 seq_num 104) dirty exists 2024-09-21T09:52:38.495 INFO:tasks.rados.rados.0.smithi033.stdout:441: finishing write tid 3 to smithi03385152-17 2024-09-21T09:52:38.495 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 151 (ObjNum 161 snap 37 seq_num 161) dirty exists 2024-09-21T09:52:38.495 INFO:tasks.rados.rados.0.smithi033.stdout:441: left oid 17 (ObjNum 161 snap 37 seq_num 161) 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:441: done (6 left) 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:443: done (5 left) 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:444: done (4 left) 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:446: done (3 left) 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:447: done (2 left) 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:449: done (1 left) 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:450: read oid 48 snap -1 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:450: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:52:38.496 INFO:tasks.rados.rados.0.smithi033.stdout:451: delete oid 10 current snap is 37 2024-09-21T09:52:38.498 INFO:tasks.rados.rados.0.smithi033.stdout:451: done (2 left) 2024-09-21T09:52:38.499 INFO:tasks.rados.rados.0.smithi033.stdout:452: copy_from oid 8 from oid 21 current snap is 37 2024-09-21T09:52:38.499 INFO:tasks.rados.rados.0.smithi033.stdout:453: delete oid 29 current snap is 37 2024-09-21T09:52:38.499 INFO:tasks.rados.rados.0.smithi033.stdout:450: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:52:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:448: finishing copy_from racing read to smithi03385152-46 2024-09-21T09:52:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:448: finishing copy_from to smithi03385152-46 2024-09-21T09:52:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:52:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:452: finishing copy_from racing read to smithi03385152-8 2024-09-21T09:52:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:448: done (3 left) 2024-09-21T09:52:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:450: done (2 left) 2024-09-21T09:52:38.521 INFO:tasks.rados.rados.0.smithi033.stdout:453: done (1 left) 2024-09-21T09:52:38.521 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 37 2024-09-21T09:52:38.521 INFO:tasks.rados.rados.0.smithi033.stdout:454: seq_num 166 ranges {5275648=385024} 2024-09-21T09:52:38.551 INFO:tasks.rados.rados.0.smithi033.stdout:454: writing smithi03385152-47 from 5275648 to 5660672 tid 1 2024-09-21T09:52:38.551 INFO:tasks.rados.rados.0.smithi033.stdout:455: rollback oid 22 current snap is 37 2024-09-21T09:52:38.551 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 22 to 22 2024-09-21T09:52:38.551 INFO:tasks.rados.rados.0.smithi033.stdout:452: finishing copy_from to smithi03385152-8 2024-09-21T09:52:38.551 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 130 (ObjNum 112 snap 25 seq_num 112) dirty exists 2024-09-21T09:52:38.551 INFO:tasks.rados.rados.0.smithi033.stdout:452: done (2 left) 2024-09-21T09:52:38.551 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 37 2024-09-21T09:52:38.552 INFO:tasks.rados.rados.0.smithi033.stdout:456: seq_num 167 ranges {0=401408} 2024-09-21T09:52:38.554 INFO:tasks.rados.rados.0.smithi033.stdout:456: writing smithi03385152-43 from 0 to 401408 tid 1 2024-09-21T09:52:38.554 INFO:tasks.rados.rados.0.smithi033.stdout:457: delete oid 5 current snap is 37 2024-09-21T09:52:38.554 INFO:tasks.rados.rados.0.smithi033.stdout:454: finishing write tid 1 to smithi03385152-47 2024-09-21T09:52:38.554 INFO:tasks.rados.rados.0.smithi033.stdout:455: finishing rollback tid 0 to smithi03385152-22 2024-09-21T09:52:38.554 INFO:tasks.rados.rados.0.smithi033.stdout:454: finishing write tid 2 to smithi03385152-47 2024-09-21T09:52:38.554 INFO:tasks.rados.rados.0.smithi033.stdout:455: finishing rollback tid 1 to smithi03385152-22 2024-09-21T09:52:38.554 INFO:tasks.rados.rados.0.smithi033.stdout:455: finishing rollback tid 2 to smithi03385152-22 2024-09-21T09:52:38.555 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 226 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:52:38.555 INFO:tasks.rados.rados.0.smithi033.stdout:454: finishing write tid 3 to smithi03385152-47 2024-09-21T09:52:38.555 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 18 (ObjNum 166 snap 37 seq_num 166) dirty exists 2024-09-21T09:52:38.556 INFO:tasks.rados.rados.0.smithi033.stdout:454: left oid 47 (ObjNum 166 snap 37 seq_num 166) 2024-09-21T09:52:38.556 INFO:tasks.rados.rados.0.smithi033.stdout:454: done (3 left) 2024-09-21T09:52:38.556 INFO:tasks.rados.rados.0.smithi033.stdout:455: done (2 left) 2024-09-21T09:52:38.556 INFO:tasks.rados.rados.0.smithi033.stdout:457: done (1 left) 2024-09-21T09:52:38.556 INFO:tasks.rados.rados.0.smithi033.stdout:458: delete oid 10 current snap is 37 2024-09-21T09:52:38.557 INFO:tasks.rados.rados.0.smithi033.stdout:456: finishing write tid 1 to smithi03385152-43 2024-09-21T09:52:38.557 INFO:tasks.rados.rados.0.smithi033.stdout:458: done (1 left) 2024-09-21T09:52:38.557 INFO:tasks.rados.rados.0.smithi033.stdout:459: rmattr oid 29 current snap is 37 2024-09-21T09:52:38.557 INFO:tasks.rados.rados.0.smithi033.stdout:459: done (1 left) 2024-09-21T09:52:38.557 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 37 2024-09-21T09:52:38.557 INFO:tasks.rados.rados.0.smithi033.stdout:460: seq_num 170 ranges {3858432=450560} 2024-09-21T09:52:38.580 INFO:tasks.rados.rados.0.smithi033.stdout:460: writing smithi03385152-37 from 3858432 to 4308992 tid 1 2024-09-21T09:52:38.580 INFO:tasks.rados.rados.0.smithi033.stdout:461: read oid 29 snap -1 2024-09-21T09:52:38.580 INFO:tasks.rados.rados.0.smithi033.stdout:461: expect deleted 2024-09-21T09:52:38.580 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 37 2024-09-21T09:52:38.580 INFO:tasks.rados.rados.0.smithi033.stdout:462: seq_num 171 ranges {2572288=466944} 2024-09-21T09:52:38.597 INFO:tasks.rados.rados.0.smithi033.stdout:462: writing smithi03385152-12 from 2572288 to 3039232 tid 1 2024-09-21T09:52:38.597 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 37 2024-09-21T09:52:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:456: finishing write tid 2 to smithi03385152-43 2024-09-21T09:52:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:456: finishing write tid 3 to smithi03385152-43 2024-09-21T09:52:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 141 (ObjNum 167 snap 37 seq_num 167) dirty exists 2024-09-21T09:52:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:456: left oid 43 (ObjNum 167 snap 37 seq_num 167) 2024-09-21T09:52:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:463: seq_num 172 ranges {3678208=483328} 2024-09-21T09:52:38.620 INFO:tasks.rados.rados.0.smithi033.stdout:463: writing smithi03385152-8 from 3678208 to 4161536 tid 1 2024-09-21T09:52:38.620 INFO:tasks.rados.rados.0.smithi033.stdout:456: done (4 left) 2024-09-21T09:52:38.620 INFO:tasks.rados.rados.0.smithi033.stdout:461: done (3 left) 2024-09-21T09:52:38.620 INFO:tasks.rados.rados.0.smithi033.stdout:464: snap_remove snap 19 2024-09-21T09:52:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:38 smithi033 ceph-mon[39045]: pgmap v207: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-21T09:52:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:38 smithi033 ceph-mon[39045]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:52:38.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:38 smithi033 ceph-mon[27969]: pgmap v207: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-21T09:52:38.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:38 smithi033 ceph-mon[27969]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:52:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:38 smithi105 ceph-mon[31286]: pgmap v207: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 6.5 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-21T09:52:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:38 smithi105 ceph-mon[31286]: osdmap e141: 8 total, 8 up, 8 in 2024-09-21T09:52:39.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:38 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:38] "GET /metrics HTTP/1.1" 200 240120 "" "Prometheus/2.43.0" 2024-09-21T09:52:39.487 INFO:tasks.rados.rados.0.smithi033.stdout:464: done (3 left) 2024-09-21T09:52:39.487 INFO:tasks.rados.rados.0.smithi033.stdout:465: snap_create 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:460: finishing write tid 1 to smithi03385152-37 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:460: finishing write tid 2 to smithi03385152-37 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:460: finishing write tid 3 to smithi03385152-37 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 143 (ObjNum 170 snap 37 seq_num 170) dirty exists 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:460: left oid 37 (ObjNum 170 snap 37 seq_num 170) 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:462: finishing write tid 1 to smithi03385152-12 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:462: finishing write tid 2 to smithi03385152-12 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:462: finishing write tid 3 to smithi03385152-12 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 154 (ObjNum 171 snap 37 seq_num 171) dirty exists 2024-09-21T09:52:39.488 INFO:tasks.rados.rados.0.smithi033.stdout:462: left oid 12 (ObjNum 171 snap 37 seq_num 171) 2024-09-21T09:52:39.489 INFO:tasks.rados.rados.0.smithi033.stdout:463: finishing write tid 1 to smithi03385152-8 2024-09-21T09:52:39.489 INFO:tasks.rados.rados.0.smithi033.stdout:463: finishing write tid 2 to smithi03385152-8 2024-09-21T09:52:39.489 INFO:tasks.rados.rados.0.smithi033.stdout:463: finishing write tid 3 to smithi03385152-8 2024-09-21T09:52:39.489 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 132 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:52:39.489 INFO:tasks.rados.rados.0.smithi033.stdout:463: left oid 8 (ObjNum 172 snap 37 seq_num 172) 2024-09-21T09:52:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:39 smithi033 ceph-mon[39045]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:52:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:39 smithi033 ceph-mon[27969]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:52:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:39 smithi105 ceph-mon[31286]: osdmap e142: 8 total, 8 up, 8 in 2024-09-21T09:52:40.491 INFO:tasks.rados.rados.0.smithi033.stdout:460: done (3 left) 2024-09-21T09:52:40.491 INFO:tasks.rados.rados.0.smithi033.stdout:462: done (2 left) 2024-09-21T09:52:40.491 INFO:tasks.rados.rados.0.smithi033.stdout:463: done (1 left) 2024-09-21T09:52:40.492 INFO:tasks.rados.rados.0.smithi033.stdout:465: done (0 left) 2024-09-21T09:52:40.492 INFO:tasks.rados.rados.0.smithi033.stdout:466: snap_remove snap 22 2024-09-21T09:52:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:40 smithi033 ceph-mon[39045]: pgmap v210: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:40 smithi033 ceph-mon[39045]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:52:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:40 smithi033 ceph-mon[27969]: pgmap v210: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:40 smithi033 ceph-mon[27969]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:52:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:40 smithi105 ceph-mon[31286]: pgmap v210: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:40 smithi105 ceph-mon[31286]: osdmap e143: 8 total, 8 up, 8 in 2024-09-21T09:52:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:466: done (0 left) 2024-09-21T09:52:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:467: read oid 16 snap 36 2024-09-21T09:52:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:467: expect (ObjNum 158 snap 34 seq_num 158) 2024-09-21T09:52:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:468: delete oid 30 current snap is 38 2024-09-21T09:52:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:468: done (1 left) 2024-09-21T09:52:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:469: read oid 4 snap 37 2024-09-21T09:52:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:469: expect deleted 2024-09-21T09:52:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:470: read oid 30 snap 29 2024-09-21T09:52:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:470: expect (ObjNum 93 snap 16 seq_num 93) 2024-09-21T09:52:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:471: read oid 40 snap 35 2024-09-21T09:52:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:471: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:52:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 38 2024-09-21T09:52:41.527 INFO:tasks.rados.rados.0.smithi033.stdout:472: seq_num 174 ranges {4308992=516096} 2024-09-21T09:52:41.553 INFO:tasks.rados.rados.0.smithi033.stdout:472: writing smithi03385152-37 from 4308992 to 4825088 tid 1 2024-09-21T09:52:41.553 INFO:tasks.rados.rados.0.smithi033.stdout:473: read oid 44 snap 35 2024-09-21T09:52:41.553 INFO:tasks.rados.rados.0.smithi033.stdout:473: expect (ObjNum 139 snap 32 seq_num 139) 2024-09-21T09:52:41.553 INFO:tasks.rados.rados.0.smithi033.stdout:474: rollback oid 6 current snap is 38 2024-09-21T09:52:41.553 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 37 2024-09-21T09:52:41.554 INFO:tasks.rados.rados.0.smithi033.stdout:467: expect (ObjNum 158 snap 34 seq_num 158) 2024-09-21T09:52:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:41 smithi105 ceph-mon[31286]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:52:41.872 INFO:tasks.rados.rados.0.smithi033.stdout:471: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:52:41.912 INFO:tasks.rados.rados.0.smithi033.stdout:470: expect (ObjNum 93 snap 16 seq_num 93) 2024-09-21T09:52:41.977 INFO:tasks.rados.rados.0.smithi033.stdout:474: finishing rollback tid 0 to smithi03385152-6 2024-09-21T09:52:41.977 INFO:tasks.rados.rados.0.smithi033.stdout:474: finishing rollback tid 1 to smithi03385152-6 2024-09-21T09:52:41.977 INFO:tasks.rados.rados.0.smithi033.stdout:472: finishing write tid 1 to smithi03385152-37 2024-09-21T09:52:41.977 INFO:tasks.rados.rados.0.smithi033.stdout:472: finishing write tid 2 to smithi03385152-37 2024-09-21T09:52:41.977 INFO:tasks.rados.rados.0.smithi033.stdout:474: finishing rollback tid 2 to smithi03385152-6 2024-09-21T09:52:41.977 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 128 (ObjNum 45 snap 0 seq_num 45) dirty exists 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:467: done (6 left) 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:469: done (5 left) 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:470: done (4 left) 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:471: done (3 left) 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:474: done (2 left) 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:475: snap_create 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:472: finishing write tid 3 to smithi03385152-37 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 145 (ObjNum 174 snap 38 seq_num 174) dirty exists 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:472: left oid 37 (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:52:41.978 INFO:tasks.rados.rados.0.smithi033.stdout:473: expect (ObjNum 139 snap 32 seq_num 139) 2024-09-21T09:52:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:41 smithi033 ceph-mon[39045]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:52:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:41 smithi033 ceph-mon[27969]: osdmap e144: 8 total, 8 up, 8 in 2024-09-21T09:52:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:472: done (2 left) 2024-09-21T09:52:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:473: done (1 left) 2024-09-21T09:52:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:475: done (0 left) 2024-09-21T09:52:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:476: read oid 22 snap -1 2024-09-21T09:52:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:476: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:52:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:477: rollback oid 47 current snap is 39 2024-09-21T09:52:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 38 2024-09-21T09:52:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:478: snap_remove snap 36 2024-09-21T09:52:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:42 smithi105 ceph-mon[31286]: pgmap v213: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:52:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:42 smithi105 ceph-mon[31286]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:52:42.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:42] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:52:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:42 smithi033 ceph-mon[39045]: pgmap v213: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:52:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:42 smithi033 ceph-mon[39045]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:52:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:42 smithi033 ceph-mon[27969]: pgmap v213: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:52:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:42 smithi033 ceph-mon[27969]: osdmap e145: 8 total, 8 up, 8 in 2024-09-21T09:52:43.517 INFO:tasks.rados.rados.0.smithi033.stdout:478: done (2 left) 2024-09-21T09:52:43.517 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 39 2024-09-21T09:52:43.517 INFO:tasks.rados.rados.0.smithi033.stdout:479: seq_num 175 ranges {761856=540672} 2024-09-21T09:52:43.524 INFO:tasks.rados.rados.0.smithi033.stdout:479: writing smithi03385152-6 from 761856 to 1302528 tid 1 2024-09-21T09:52:43.524 INFO:tasks.rados.rados.0.smithi033.stdout:480: copy_from oid 41 from oid 46 current snap is 39 2024-09-21T09:52:43.525 INFO:tasks.rados.rados.0.smithi033.stdout:477: finishing rollback tid 0 to smithi03385152-47 2024-09-21T09:52:43.525 INFO:tasks.rados.rados.0.smithi033.stdout:477: finishing rollback tid 1 to smithi03385152-47 2024-09-21T09:52:43.525 INFO:tasks.rados.rados.0.smithi033.stdout:477: finishing rollback tid 2 to smithi03385152-47 2024-09-21T09:52:43.525 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 26 (ObjNum 166 snap 37 seq_num 166) dirty exists 2024-09-21T09:52:43.525 INFO:tasks.rados.rados.0.smithi033.stdout:476: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:52:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:480: finishing copy_from racing read to smithi03385152-41 2024-09-21T09:52:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:480: finishing copy_from to smithi03385152-41 2024-09-21T09:52:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:52:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:479: finishing write tid 1 to smithi03385152-6 2024-09-21T09:52:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:479: finishing write tid 2 to smithi03385152-6 2024-09-21T09:52:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:479: finishing write tid 3 to smithi03385152-6 2024-09-21T09:52:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 135 (ObjNum 175 snap 39 seq_num 175) dirty exists 2024-09-21T09:52:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:479: left oid 6 (ObjNum 175 snap 39 seq_num 175) 2024-09-21T09:52:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:476: done (3 left) 2024-09-21T09:52:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:477: done (2 left) 2024-09-21T09:52:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:479: done (1 left) 2024-09-21T09:52:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:480: done (0 left) 2024-09-21T09:52:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:481: rollback oid 1 current snap is 39 2024-09-21T09:52:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 30 2024-09-21T09:52:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:482: snap_remove snap 35 2024-09-21T09:52:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:43 smithi105 ceph-mon[31286]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:52:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:43 smithi033 ceph-mon[39045]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:52:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:43 smithi033 ceph-mon[27969]: osdmap e146: 8 total, 8 up, 8 in 2024-09-21T09:52:44.533 INFO:tasks.rados.rados.0.smithi033.stdout:482: done (1 left) 2024-09-21T09:52:44.533 INFO:tasks.rados.rados.0.smithi033.stdout:483: snap_remove snap 37 2024-09-21T09:52:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:44 smithi105 ceph-mon[31286]: pgmap v216: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:52:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:44 smithi105 ceph-mon[31286]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:52:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:44 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:44.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:44 smithi033 ceph-mon[39045]: pgmap v216: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:52:44.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:44 smithi033 ceph-mon[39045]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:52:44.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:44 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:44.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:44 smithi033 ceph-mon[27969]: pgmap v216: 121 pgs: 121 active+clean; 134 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.2 MiB/s wr, 12 op/s 2024-09-21T09:52:44.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:44 smithi033 ceph-mon[27969]: osdmap e147: 8 total, 8 up, 8 in 2024-09-21T09:52:44.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:44 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:45.546 INFO:tasks.rados.rados.0.smithi033.stdout:483: done (1 left) 2024-09-21T09:52:45.546 INFO:tasks.rados.rados.0.smithi033.stdout:484: snap_create 2024-09-21T09:52:45.546 INFO:tasks.rados.rados.0.smithi033.stdout:481: finishing rollback tid 0 to smithi03385152-1 2024-09-21T09:52:45.546 INFO:tasks.rados.rados.0.smithi033.stdout:481: finishing rollback tid 1 to smithi03385152-1 2024-09-21T09:52:45.547 INFO:tasks.rados.rados.0.smithi033.stdout:481: finishing rollback tid 2 to smithi03385152-1 2024-09-21T09:52:45.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 135 (ObjNum 33 snap 0 seq_num 91102320) dirty exists 2024-09-21T09:52:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:45 smithi105 ceph-mon[31286]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:52:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:45 smithi033 ceph-mon[39045]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:52:48.080 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:45 smithi033 ceph-mon[27969]: osdmap e148: 8 total, 8 up, 8 in 2024-09-21T09:52:48.081 INFO:tasks.rados.rados.0.smithi033.stdout:481: done (1 left) 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:484: done (0 left) 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:485: read oid 47 snap -1 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:485: expect (ObjNum 166 snap 37 seq_num 166) 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:486: delete oid 33 current snap is 40 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:486: done (1 left) 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:487: rollback oid 8 current snap is 40 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 8 to 38 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 40 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:488: seq_num 177 ranges {3039232=573440} 2024-09-21T09:52:48.082 INFO:tasks.rados.rados.0.smithi033.stdout:488: writing smithi03385152-12 from 3039232 to 3612672 tid 1 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 40 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:489: seq_num 178 ranges {3973120=589824} 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:489: writing smithi03385152-31 from 3973120 to 4562944 tid 1 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:490: rollback oid 41 current snap is 40 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 38 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:487: finishing rollback tid 0 to smithi03385152-8 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:487: finishing rollback tid 1 to smithi03385152-8 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:487: finishing rollback tid 2 to smithi03385152-8 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 138 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:52:48.083 INFO:tasks.rados.rados.0.smithi033.stdout:485: expect (ObjNum 166 snap 37 seq_num 166) 2024-09-21T09:52:48.084 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ignoring --setuser ceph since I am not root 2024-09-21T09:52:48.084 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ignoring --setgroup ceph since I am not root 2024-09-21T09:52:48.084 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mgr[28245]: -- 172.21.15.33:0/1297139766 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5646b57de340 con 0x5646b57eac00 2024-09-21T09:52:48.084 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:46.604+0000 7f0a5fc5fec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-21T09:52:48.084 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:46.661+0000 7f0a5fc5fec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-21T09:52:48.084 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:46.729+0000 7f0a5fc5fec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: pgmap v219: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 264 KiB/s wr, 8 op/s 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:48.085 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: Activating manager daemon x 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: mgrmap e24: x(active, starting, since 0.00766292s) 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:52:48.086 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:52:48.087 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:52:48.088 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[39045]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:52:48.088 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: pgmap v219: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 264 KiB/s wr, 8 op/s 2024-09-21T09:52:48.088 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: Activating manager daemon x 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: mgrmap e24: x(active, starting, since 0.00766292s) 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:48.089 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:52:48.090 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:52:48.091 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:52:48.091 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:52:48.091 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:52:48.091 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:52:48.091 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:52:48.091 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-mon[27969]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:52:48.092 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:46] ENGINE Bus STOPPING 2024-09-21T09:52:48.093 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: pgmap v219: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 264 KiB/s wr, 8 op/s 2024-09-21T09:52:48.093 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: osdmap e149: 8 total, 8 up, 8 in 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: Activating manager daemon x 2024-09-21T09:52:48.094 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: osdmap e150: 8 total, 8 up, 8 in 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.14550 172.21.15.33:0/3414204005' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: mgrmap e24: x(active, starting, since 0.00766292s) 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:52:48.095 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:52:48.096 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:52:48.096 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:52:48.096 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:52:48.096 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:52:48.096 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:52:48.096 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-mon[31286]: osdmap e151: 8 total, 8 up, 8 in 2024-09-21T09:52:48.096 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:52:48.096 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:46] ENGINE Bus STOPPED 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:488: finishing write tid 1 to smithi03385152-12 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:488: finishing write tid 2 to smithi03385152-12 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:488: finishing write tid 3 to smithi03385152-12 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 181 (ObjNum 177 snap 40 seq_num 177) dirty exists 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:488: left oid 12 (ObjNum 177 snap 40 seq_num 177) 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:485: done (4 left) 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:487: done (3 left) 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:488: done (2 left) 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:491: read oid 38 snap -1 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:491: expect (ObjNum 155 snap 34 seq_num 155) 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 40 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:490: finishing rollback tid 0 to smithi03385152-41 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:490: finishing rollback tid 1 to smithi03385152-41 2024-09-21T09:52:48.097 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 159 (ObjNum 289 snap 0 seq_num 1546671879) dirty dne 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:489: finishing write tid 1 to smithi03385152-31 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:489: finishing write tid 2 to smithi03385152-31 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:489: finishing write tid 3 to smithi03385152-31 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 327 (ObjNum 178 snap 40 seq_num 178) dirty exists 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:489: left oid 31 (ObjNum 178 snap 40 seq_num 178) 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:492: seq_num 179 ranges {1794048=606208} 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:492: writing smithi03385152-32 from 1794048 to 2400256 tid 1 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:489: done (3 left) 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:490: done (2 left) 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:493: snap_create 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:491: expect (ObjNum 155 snap 34 seq_num 155) 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:492: finishing write tid 1 to smithi03385152-32 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:492: finishing write tid 2 to smithi03385152-32 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:492: finishing write tid 3 to smithi03385152-32 2024-09-21T09:52:48.098 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 41 (ObjNum 179 snap 40 seq_num 179) dirty exists 2024-09-21T09:52:48.099 INFO:tasks.rados.rados.0.smithi033.stdout:492: left oid 32 (ObjNum 179 snap 40 seq_num 179) 2024-09-21T09:52:48.099 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:46.864+0000 7f0a5fc5fec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:52:48.099 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:47.008+0000 7f0a5fc5fec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-21T09:52:48.099 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:47.274+0000 7f0a5fc5fec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-21T09:52:48.099 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:47.336+0000 7f0a5fc5fec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-21T09:52:48.099 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:47.505+0000 7f0a5fc5fec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-mon[31286]: Manager daemon x is now available 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:47] ENGINE Bus STARTING 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.644+0000 7f7c681b8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.644+0000 7f7c681b8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.644+0000 7f7c681b8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.644+0000 7f7c681b8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.644+0000 7f7c681b8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.646+0000 7f7c6b1be640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.100 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.646+0000 7f7c6b1be640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.101 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.646+0000 7f7c6b1be640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.101 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.646+0000 7f7c6b1be640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.101 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:47.646+0000 7f7c6b1be640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:52:48.101 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:47] ENGINE Serving on http://:::9283 2024-09-21T09:52:48.101 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:52:47] ENGINE Bus STARTED 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:491: done (2 left) 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:492: done (1 left) 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:493: done (0 left) 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 41 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:494: seq_num 180 ranges {1941504=622592} 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:494: writing smithi03385152-23 from 1941504 to 2564096 tid 1 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:495: copy_from oid 26 from oid 22 current snap is 41 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:496: snap_create 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:495: finishing copy_from racing read to smithi03385152-26 2024-09-21T09:52:48.102 INFO:tasks.rados.rados.0.smithi033.stdout:494: finishing write tid 1 to smithi03385152-23 2024-09-21T09:52:48.103 INFO:tasks.rados.rados.0.smithi033.stdout:494: finishing write tid 2 to smithi03385152-23 2024-09-21T09:52:48.103 INFO:tasks.rados.rados.0.smithi033.stdout:495: finishing copy_from to smithi03385152-26 2024-09-21T09:52:48.103 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 226 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:52:48.103 INFO:tasks.rados.rados.0.smithi033.stdout:494: finishing write tid 3 to smithi03385152-23 2024-09-21T09:52:48.103 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 173 (ObjNum 180 snap 41 seq_num 180) dirty exists 2024-09-21T09:52:48.103 INFO:tasks.rados.rados.0.smithi033.stdout:494: left oid 23 (ObjNum 180 snap 41 seq_num 180) 2024-09-21T09:52:48.103 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:47 smithi033 ceph-mon[39045]: Manager daemon x is now available 2024-09-21T09:52:48.104 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:47 smithi033 ceph-mon[27969]: Manager daemon x is now available 2024-09-21T09:52:48.565 INFO:tasks.rados.rados.0.smithi033.stdout:494: done (2 left) 2024-09-21T09:52:48.565 INFO:tasks.rados.rados.0.smithi033.stdout:495: done (1 left) 2024-09-21T09:52:48.565 INFO:tasks.rados.rados.0.smithi033.stdout:496: done (0 left) 2024-09-21T09:52:48.565 INFO:tasks.rados.rados.0.smithi033.stdout:497: snap_create 2024-09-21T09:52:48.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:48.415+0000 7f0a5fc5fec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-21T09:52:48.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:48.483+0000 7f0a5fc5fec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: mgrmap e25: x(active, since 1.10747s) 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: Checking for cert/key for grafana.a 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: Migrating grafana.a cert to cert store 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: Migrating grafana.a key to cert store 2024-09-21T09:52:48.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:52:48.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: mgrmap e25: x(active, since 1.10747s) 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: Checking for cert/key for grafana.a 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: Migrating grafana.a cert to cert store 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: Migrating grafana.a key to cert store 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:52:48.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:52:48.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:52:48.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:52:48.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:52:48.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:48 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:52:49.013 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: mgrmap e25: x(active, since 1.10747s) 2024-09-21T09:52:49.013 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: osdmap e152: 8 total, 8 up, 8 in 2024-09-21T09:52:49.013 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:49.013 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:49.013 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: Checking for cert/key for grafana.a 2024-09-21T09:52:49.013 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: Migrating grafana.a cert to cert store 2024-09-21T09:52:49.013 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:49.013 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: Migrating grafana.a key to cert store 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:52:49.014 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:48 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-21T09:52:49.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:49.010+0000 7f7c53f13640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:52:49] ENGINE Error in HTTPServer.serve 2024-09-21T09:52:49.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: Traceback (most recent call last): 2024-09-21T09:52:49.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:52:49.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: self._connections.run(self.expiration_interval) 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: self._run(expiration_interval) 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: s = self.context.wrap_socket( 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: return self.sslsocket_class._create( 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: self.do_handshake() 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:52:49.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: self._sslobj.do_handshake() 2024-09-21T09:52:49.282 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:52:49.282 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:52:49.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:49.242+0000 7f0a5fc5fec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-21T09:52:49.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:49.359+0000 7f0a5fc5fec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-21T09:52:49.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:49.484+0000 7f0a5fc5fec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-21T09:52:49.566 INFO:tasks.rados.rados.0.smithi033.stdout:497: done (0 left) 2024-09-21T09:52:49.566 INFO:tasks.rados.rados.0.smithi033.stdout:498: delete oid 4 current snap is 43 2024-09-21T09:52:49.571 INFO:tasks.rados.rados.0.smithi033.stdout:498: done (0 left) 2024-09-21T09:52:49.571 INFO:tasks.rados.rados.0.smithi033.stdout:499: delete oid 15 current snap is 43 2024-09-21T09:52:49.575 INFO:tasks.rados.rados.0.smithi033.stdout:499: done (0 left) 2024-09-21T09:52:49.575 INFO:tasks.rados.rados.0.smithi033.stdout:500: snap_create 2024-09-21T09:52:49.823 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-mon[39045]: Deploying cephadm binary to smithi033 2024-09-21T09:52:49.823 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-mon[39045]: pgmap v6: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:49.824 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-mon[39045]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:52:49.824 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-mon[39045]: Deploying cephadm binary to smithi105 2024-09-21T09:52:49.824 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-mon[27969]: Deploying cephadm binary to smithi033 2024-09-21T09:52:49.824 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-mon[27969]: pgmap v6: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:49.824 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-mon[27969]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:52:49.824 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-mon[27969]: Deploying cephadm binary to smithi105 2024-09-21T09:52:49.825 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:49.547+0000 7f0a5fc5fec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-21T09:52:49.825 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:49.687+0000 7f0a5fc5fec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-21T09:52:49.825 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:49.758+0000 7f0a5fc5fec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-21T09:52:49.825 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:49.822+0000 7f0a5fc5fec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-21T09:52:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-mon[31286]: Deploying cephadm binary to smithi033 2024-09-21T09:52:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-mon[31286]: pgmap v6: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-mon[31286]: osdmap e153: 8 total, 8 up, 8 in 2024-09-21T09:52:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:49 smithi105 ceph-mon[31286]: Deploying cephadm binary to smithi105 2024-09-21T09:52:50.124 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:49.986+0000 7f0a5fc5fec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-21T09:52:50.124 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:50.054+0000 7f0a5fc5fec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-21T09:52:50.125 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:50.122+0000 7f0a5fc5fec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-21T09:52:50.382 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:50.208+0000 7f0a5fc5fec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-21T09:52:50.382 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:50.273+0000 7f0a5fc5fec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-21T09:52:50.382 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:50.335+0000 7f0a5fc5fec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-21T09:52:50.577 INFO:tasks.rados.rados.0.smithi033.stdout:500: done (0 left) 2024-09-21T09:52:50.577 INFO:tasks.rados.rados.0.smithi033.stdout:501: setattr oid 19 current snap is 44 2024-09-21T09:52:50.578 INFO:tasks.rados.rados.0.smithi033.stdout:502: rmattr oid 42 current snap is 44 2024-09-21T09:52:50.578 INFO:tasks.rados.rados.0.smithi033.stdout:502: done (1 left) 2024-09-21T09:52:50.579 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 44 2024-09-21T09:52:50.579 INFO:tasks.rados.rados.0.smithi033.stdout:503: seq_num 183 ranges {638976=671744} 2024-09-21T09:52:50.588 INFO:tasks.rados.rados.0.smithi033.stdout:503: writing smithi03385152-39 from 638976 to 1310720 tid 1 2024-09-21T09:52:50.588 INFO:tasks.rados.rados.0.smithi033.stdout:504: read oid 47 snap -1 2024-09-21T09:52:50.588 INFO:tasks.rados.rados.0.smithi033.stdout:504: expect (ObjNum 166 snap 37 seq_num 166) 2024-09-21T09:52:50.588 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 44 2024-09-21T09:52:50.588 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 73 (ObjNum 33 snap 0 seq_num 90883040) dirty exists 2024-09-21T09:52:50.588 INFO:tasks.rados.rados.0.smithi033.stdout:505: seq_num 184 ranges {0=688128} 2024-09-21T09:52:50.593 INFO:tasks.rados.rados.0.smithi033.stdout:505: writing smithi03385152-41 from 0 to 688128 tid 1 2024-09-21T09:52:50.593 INFO:tasks.rados.rados.0.smithi033.stdout:501: done (3 left) 2024-09-21T09:52:50.593 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 44 2024-09-21T09:52:50.593 INFO:tasks.rados.rados.0.smithi033.stdout:506: seq_num 185 ranges {2564096=704512} 2024-09-21T09:52:50.614 INFO:tasks.rados.rados.0.smithi033.stdout:506: writing smithi03385152-23 from 2564096 to 3268608 tid 1 2024-09-21T09:52:50.614 INFO:tasks.rados.rados.0.smithi033.stdout:507: read oid 30 snap -1 2024-09-21T09:52:50.614 INFO:tasks.rados.rados.0.smithi033.stdout:507: expect deleted 2024-09-21T09:52:50.614 INFO:tasks.rados.rados.0.smithi033.stdout:508: copy_from oid 11 from oid 2 current snap is 44 2024-09-21T09:52:50.614 INFO:tasks.rados.rados.0.smithi033.stdout:503: finishing write tid 1 to smithi03385152-39 2024-09-21T09:52:50.614 INFO:tasks.rados.rados.0.smithi033.stdout:503: finishing write tid 2 to smithi03385152-39 2024-09-21T09:52:50.614 INFO:tasks.rados.rados.0.smithi033.stdout:503: finishing write tid 3 to smithi03385152-39 2024-09-21T09:52:50.615 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 55 (ObjNum 183 snap 44 seq_num 183) dirty exists 2024-09-21T09:52:50.615 INFO:tasks.rados.rados.0.smithi033.stdout:503: left oid 39 (ObjNum 183 snap 44 seq_num 183) 2024-09-21T09:52:50.615 INFO:tasks.rados.rados.0.smithi033.stdout:505: finishing write tid 1 to smithi03385152-41 2024-09-21T09:52:50.615 INFO:tasks.rados.rados.0.smithi033.stdout:505: finishing write tid 2 to smithi03385152-41 2024-09-21T09:52:50.615 INFO:tasks.rados.rados.0.smithi033.stdout:504: expect (ObjNum 166 snap 37 seq_num 166) 2024-09-21T09:52:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: [21/Sep/2024:09:52:48] ENGINE Bus STARTING 2024-09-21T09:52:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: [21/Sep/2024:09:52:48] ENGINE Serving on https://172.21.15.105:7150 2024-09-21T09:52:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: [21/Sep/2024:09:52:49] ENGINE Error in HTTPServer.serve 2024-09-21T09:52:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:52:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:52:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: self.do_handshake() 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:52:50.747 INFO:journalctl@ceph.mon.a.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: [21/Sep/2024:09:52:49] ENGINE Serving on http://172.21.15.105:8765 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: [21/Sep/2024:09:52:49] ENGINE Bus STARTED 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: mgrmap e26: x(active, since 3s) 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: [21/Sep/2024:09:52:48] ENGINE Bus STARTING 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: [21/Sep/2024:09:52:48] ENGINE Serving on https://172.21.15.105:7150 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: [21/Sep/2024:09:52:49] ENGINE Error in HTTPServer.serve 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.c.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:52:50.748 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: self.do_handshake() 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:52:50.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: [21/Sep/2024:09:52:49] ENGINE Serving on http://172.21.15.105:8765 2024-09-21T09:52:50.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: [21/Sep/2024:09:52:49] ENGINE Bus STARTED 2024-09-21T09:52:50.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:52:50.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: mgrmap e26: x(active, since 3s) 2024-09-21T09:52:50.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:50.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:50 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:505: finishing write tid 3 to smithi03385152-41 2024-09-21T09:52:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 161 (ObjNum 184 snap 44 seq_num 184) dirty exists 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:505: left oid 41 (ObjNum 184 snap 44 seq_num 184) 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:508: finishing copy_from racing read to smithi03385152-11 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:506: finishing write tid 1 to smithi03385152-23 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:503: done (5 left) 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:504: done (4 left) 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:505: done (3 left) 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:507: done (2 left) 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 44 2024-09-21T09:52:50.924 INFO:tasks.rados.rados.0.smithi033.stdout:509: seq_num 186 ranges {6234112=720896} 2024-09-21T09:52:50.963 INFO:tasks.rados.rados.0.smithi033.stdout:509: writing smithi03385152-14 from 6234112 to 6955008 tid 1 2024-09-21T09:52:50.963 INFO:tasks.rados.rados.0.smithi033.stdout:510: delete oid 16 current snap is 44 2024-09-21T09:52:50.963 INFO:tasks.rados.rados.0.smithi033.stdout:508: finishing copy_from to smithi03385152-11 2024-09-21T09:52:50.963 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 2 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:52:50.963 INFO:tasks.rados.rados.0.smithi033.stdout:506: finishing write tid 2 to smithi03385152-23 2024-09-21T09:52:50.963 INFO:tasks.rados.rados.0.smithi033.stdout:506: finishing write tid 3 to smithi03385152-23 2024-09-21T09:52:50.963 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 176 (ObjNum 185 snap 44 seq_num 185) dirty exists 2024-09-21T09:52:50.963 INFO:tasks.rados.rados.0.smithi033.stdout:506: left oid 23 (ObjNum 185 snap 44 seq_num 185) 2024-09-21T09:52:50.966 INFO:tasks.rados.rados.0.smithi033.stdout:509: finishing write tid 1 to smithi03385152-14 2024-09-21T09:52:50.966 INFO:tasks.rados.rados.0.smithi033.stdout:509: finishing write tid 2 to smithi03385152-14 2024-09-21T09:52:50.967 INFO:tasks.rados.rados.0.smithi033.stdout:509: finishing write tid 3 to smithi03385152-14 2024-09-21T09:52:50.967 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 259 (ObjNum 186 snap 44 seq_num 186) dirty exists 2024-09-21T09:52:50.967 INFO:tasks.rados.rados.0.smithi033.stdout:509: left oid 14 (ObjNum 186 snap 44 seq_num 186) 2024-09-21T09:52:50.968 INFO:tasks.rados.rados.0.smithi033.stdout:506: done (3 left) 2024-09-21T09:52:50.968 INFO:tasks.rados.rados.0.smithi033.stdout:508: done (2 left) 2024-09-21T09:52:50.968 INFO:tasks.rados.rados.0.smithi033.stdout:509: done (1 left) 2024-09-21T09:52:50.968 INFO:tasks.rados.rados.0.smithi033.stdout:510: done (0 left) 2024-09-21T09:52:50.968 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 44 2024-09-21T09:52:50.968 INFO:tasks.rados.rados.0.smithi033.stdout:511: seq_num 188 ranges {3284992=753664} 2024-09-21T09:52:50.990 INFO:tasks.rados.rados.0.smithi033.stdout:511: writing smithi03385152-38 from 3284992 to 4038656 tid 1 2024-09-21T09:52:50.990 INFO:tasks.rados.rados.0.smithi033.stdout:512: setattr oid 26 current snap is 44 2024-09-21T09:52:50.991 INFO:tasks.rados.rados.0.smithi033.stdout:513: delete oid 43 current snap is 44 2024-09-21T09:52:50.994 INFO:tasks.rados.rados.0.smithi033.stdout:511: finishing write tid 1 to smithi03385152-38 2024-09-21T09:52:50.994 INFO:tasks.rados.rados.0.smithi033.stdout:511: finishing write tid 2 to smithi03385152-38 2024-09-21T09:52:50.994 INFO:tasks.rados.rados.0.smithi033.stdout:513: done (2 left) 2024-09-21T09:52:50.994 INFO:tasks.rados.rados.0.smithi033.stdout:514: snap_remove snap 38 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: [21/Sep/2024:09:52:48] ENGINE Bus STARTING 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: [21/Sep/2024:09:52:48] ENGINE Serving on https://172.21.15.105:7150 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: [21/Sep/2024:09:52:49] ENGINE Error in HTTPServer.serve 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:52:51.027 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: [21/Sep/2024:09:52:49] ENGINE Serving on http://172.21.15.105:8765 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: [21/Sep/2024:09:52:49] ENGINE Bus STARTED 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: osdmap e154: 8 total, 8 up, 8 in 2024-09-21T09:52:51.028 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: mgrmap e26: x(active, since 3s) 2024-09-21T09:52:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:51.029 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:50 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:51.482 INFO:tasks.rados.rados.0.smithi033.stdout:514: done (2 left) 2024-09-21T09:52:51.482 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 44 2024-09-21T09:52:51.483 INFO:tasks.rados.rados.0.smithi033.stdout:515: seq_num 190 ranges {0=786432} 2024-09-21T09:52:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:515: writing smithi03385152-33 from 0 to 786432 tid 1 2024-09-21T09:52:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 227 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:52:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:511: finishing write tid 3 to smithi03385152-38 2024-09-21T09:52:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 164 (ObjNum 188 snap 44 seq_num 188) dirty exists 2024-09-21T09:52:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:511: left oid 38 (ObjNum 188 snap 44 seq_num 188) 2024-09-21T09:52:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:511: done (2 left) 2024-09-21T09:52:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:512: done (1 left) 2024-09-21T09:52:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:516: snap_create 2024-09-21T09:52:51.492 INFO:tasks.rados.rados.0.smithi033.stdout:515: finishing write tid 1 to smithi03385152-33 2024-09-21T09:52:51.492 INFO:tasks.rados.rados.0.smithi033.stdout:515: finishing write tid 2 to smithi03385152-33 2024-09-21T09:52:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:515: finishing write tid 3 to smithi03385152-33 2024-09-21T09:52:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 149 (ObjNum 190 snap 44 seq_num 190) dirty exists 2024-09-21T09:52:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:515: left oid 33 (ObjNum 190 snap 44 seq_num 190) 2024-09-21T09:52:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:51.233+0000 7f0a5fc5fec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-21T09:52:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:52:51] ENGINE Bus STARTING 2024-09-21T09:52:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: CherryPy Checker: 2024-09-21T09:52:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: The Application mounted at '' has an empty config. 2024-09-21T09:52:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: 2024-09-21T09:52:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:52:51] ENGINE Serving on http://:::9283 2024-09-21T09:52:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: [21/Sep/2024:09:52:51] ENGINE Bus STARTED 2024-09-21T09:52:51.947 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:51 smithi105 ceph-mon[31286]: pgmap v9: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:51.947 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:51 smithi105 ceph-mon[31286]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:52:51.947 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:51 smithi105 ceph-mon[31286]: Standby manager daemon y started 2024-09-21T09:52:51.948 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:51 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:52:51.948 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:51 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:51.948 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:51 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:52:51.948 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:51 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:51.948 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:51 smithi105 ceph-mon[31286]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:52:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[27969]: pgmap v9: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[27969]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:52:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[27969]: Standby manager daemon y started 2024-09-21T09:52:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:52:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[27969]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[39045]: pgmap v9: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[39045]: osdmap e155: 8 total, 8 up, 8 in 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[39045]: Standby manager daemon y started 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.33:0/3191891221' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:52:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:51 smithi033 ceph-mon[39045]: osdmap e156: 8 total, 8 up, 8 in 2024-09-21T09:52:52.479 INFO:tasks.rados.rados.0.smithi033.stdout:515: done (1 left) 2024-09-21T09:52:52.479 INFO:tasks.rados.rados.0.smithi033.stdout:516: done (0 left) 2024-09-21T09:52:52.479 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 45 2024-09-21T09:52:52.479 INFO:tasks.rados.rados.0.smithi033.stdout:517: seq_num 191 ranges {1720320=802816} 2024-09-21T09:52:52.493 INFO:tasks.rados.rados.0.smithi033.stdout:517: writing smithi03385152-27 from 1720320 to 2523136 tid 1 2024-09-21T09:52:52.493 INFO:tasks.rados.rados.0.smithi033.stdout:518: rollback oid 16 current snap is 45 2024-09-21T09:52:52.493 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 16 to 44 2024-09-21T09:52:52.493 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 45 2024-09-21T09:52:52.493 INFO:tasks.rados.rados.0.smithi033.stdout:519: seq_num 192 ranges {2400256=417792,2818048=401408} 2024-09-21T09:52:52.508 INFO:tasks.rados.rados.0.smithi033.stdout:519: writing smithi03385152-32 from 2400256 to 2818048 tid 1 2024-09-21T09:52:52.511 INFO:tasks.rados.rados.0.smithi033.stdout:519: writing smithi03385152-32 from 2818048 to 3219456 tid 2 2024-09-21T09:52:52.511 INFO:tasks.rados.rados.0.smithi033.stdout:520: snap_create 2024-09-21T09:52:52.511 INFO:tasks.rados.rados.0.smithi033.stdout:518: finishing rollback tid 1 to smithi03385152-16 2024-09-21T09:52:52.511 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 329 (ObjNum 33 snap 0 seq_num 90945120) dirty dne 2024-09-21T09:52:52.511 INFO:tasks.rados.rados.0.smithi033.stdout:517: finishing write tid 1 to smithi03385152-27 2024-09-21T09:52:52.511 INFO:tasks.rados.rados.0.smithi033.stdout:517: finishing write tid 2 to smithi03385152-27 2024-09-21T09:52:52.511 INFO:tasks.rados.rados.0.smithi033.stdout:517: finishing write tid 3 to smithi03385152-27 2024-09-21T09:52:52.512 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 152 (ObjNum 191 snap 45 seq_num 191) dirty exists 2024-09-21T09:52:52.512 INFO:tasks.rados.rados.0.smithi033.stdout:517: left oid 27 (ObjNum 191 snap 45 seq_num 191) 2024-09-21T09:52:52.512 INFO:tasks.rados.rados.0.smithi033.stdout:519: finishing write tid 1 to smithi03385152-32 2024-09-21T09:52:52.512 INFO:tasks.rados.rados.0.smithi033.stdout:519: finishing write tid 2 to smithi03385152-32 2024-09-21T09:52:52.512 INFO:tasks.rados.rados.0.smithi033.stdout:519: finishing write tid 3 to smithi03385152-32 2024-09-21T09:52:52.513 INFO:tasks.rados.rados.0.smithi033.stdout:519: finishing write tid 4 to smithi03385152-32 2024-09-21T09:52:52.513 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 47 (ObjNum 192 snap 45 seq_num 192) dirty exists 2024-09-21T09:52:52.513 INFO:tasks.rados.rados.0.smithi033.stdout:519: left oid 32 (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:52:52.629 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:52:52 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:52] "GET /metrics HTTP/1.1" 200 35928 "" "Prometheus/2.43.0" 2024-09-21T09:52:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:52 smithi033 ceph-mon[27969]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-21T09:52:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:52 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:52:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:52 smithi033 ceph-mon[27969]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:52:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:52 smithi033 ceph-mon[39045]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-21T09:52:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:52 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:52:52.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:52 smithi033 ceph-mon[39045]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:52:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:52 smithi105 ceph-mon[31286]: mgrmap e27: x(active, since 5s), standbys: y 2024-09-21T09:52:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:52 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:52:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:52 smithi105 ceph-mon[31286]: osdmap e157: 8 total, 8 up, 8 in 2024-09-21T09:52:53.479 INFO:tasks.rados.rados.0.smithi033.stdout:517: done (3 left) 2024-09-21T09:52:53.479 INFO:tasks.rados.rados.0.smithi033.stdout:518: done (2 left) 2024-09-21T09:52:53.479 INFO:tasks.rados.rados.0.smithi033.stdout:519: done (1 left) 2024-09-21T09:52:53.479 INFO:tasks.rados.rados.0.smithi033.stdout:520: done (0 left) 2024-09-21T09:52:53.480 INFO:tasks.rados.rados.0.smithi033.stdout:521: setattr oid 7 current snap is 46 2024-09-21T09:52:53.480 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 46 2024-09-21T09:52:53.480 INFO:tasks.rados.rados.0.smithi033.stdout:522: seq_num 193 ranges {0=434176,434176=401408} 2024-09-21T09:52:53.482 INFO:tasks.rados.rados.0.smithi033.stdout:522: writing smithi03385152-19 from 0 to 434176 tid 1 2024-09-21T09:52:53.485 INFO:tasks.rados.rados.0.smithi033.stdout:522: writing smithi03385152-19 from 434176 to 835584 tid 2 2024-09-21T09:52:53.485 INFO:tasks.rados.rados.0.smithi033.stdout:523: copy_from oid 13 from oid 8 current snap is 46 2024-09-21T09:52:53.485 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 335 (ObjNum 33 snap 0 seq_num 1747984172) dirty exists 2024-09-21T09:52:53.485 INFO:tasks.rados.rados.0.smithi033.stdout:521: done (2 left) 2024-09-21T09:52:53.486 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 46 2024-09-21T09:52:53.486 INFO:tasks.rados.rados.0.smithi033.stdout:524: seq_num 194 ranges {688128=458752,1146880=401408} 2024-09-21T09:52:53.492 INFO:tasks.rados.rados.0.smithi033.stdout:524: writing smithi03385152-41 from 688128 to 1146880 tid 1 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:524: writing smithi03385152-41 from 1146880 to 1548288 tid 2 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:522: finishing write tid 1 to smithi03385152-19 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:522: finishing write tid 2 to smithi03385152-19 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:522: finishing write tid 3 to smithi03385152-19 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:525: read oid 20 snap 29 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:525: expect deleted 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:526: read oid 21 snap -1 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:526: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:527: read oid 7 snap -1 2024-09-21T09:52:53.495 INFO:tasks.rados.rados.0.smithi033.stdout:527: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:52:53.496 INFO:tasks.rados.rados.0.smithi033.stdout:528: delete oid 44 current snap is 46 2024-09-21T09:52:53.496 INFO:tasks.rados.rados.0.smithi033.stdout:522: finishing write tid 4 to smithi03385152-19 2024-09-21T09:52:53.496 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 77 (ObjNum 193 snap 46 seq_num 193) dirty exists 2024-09-21T09:52:53.496 INFO:tasks.rados.rados.0.smithi033.stdout:522: left oid 19 (ObjNum 193 snap 46 seq_num 193) 2024-09-21T09:52:53.496 INFO:tasks.rados.rados.0.smithi033.stdout:524: finishing write tid 1 to smithi03385152-41 2024-09-21T09:52:53.496 INFO:tasks.rados.rados.0.smithi033.stdout:524: finishing write tid 2 to smithi03385152-41 2024-09-21T09:52:53.497 INFO:tasks.rados.rados.0.smithi033.stdout:524: finishing write tid 3 to smithi03385152-41 2024-09-21T09:52:53.498 INFO:tasks.rados.rados.0.smithi033.stdout:524: finishing write tid 4 to smithi03385152-41 2024-09-21T09:52:53.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 170 (ObjNum 194 snap 46 seq_num 194) dirty exists 2024-09-21T09:52:53.499 INFO:tasks.rados.rados.0.smithi033.stdout:524: left oid 41 (ObjNum 194 snap 46 seq_num 194) 2024-09-21T09:52:53.499 INFO:tasks.rados.rados.0.smithi033.stdout:522: done (6 left) 2024-09-21T09:52:53.499 INFO:tasks.rados.rados.0.smithi033.stdout:524: done (5 left) 2024-09-21T09:52:53.499 INFO:tasks.rados.rados.0.smithi033.stdout:525: done (4 left) 2024-09-21T09:52:53.499 INFO:tasks.rados.rados.0.smithi033.stdout:528: done (3 left) 2024-09-21T09:52:53.499 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 46 2024-09-21T09:52:53.499 INFO:tasks.rados.rados.0.smithi033.stdout:529: seq_num 196 ranges {1302528=491520,1794048=401408} 2024-09-21T09:52:53.508 INFO:tasks.rados.rados.0.smithi033.stdout:529: writing smithi03385152-6 from 1302528 to 1794048 tid 1 2024-09-21T09:52:53.511 INFO:tasks.rados.rados.0.smithi033.stdout:529: writing smithi03385152-6 from 1794048 to 2195456 tid 2 2024-09-21T09:52:53.511 INFO:tasks.rados.rados.0.smithi033.stdout:530: read oid 31 snap -1 2024-09-21T09:52:53.511 INFO:tasks.rados.rados.0.smithi033.stdout:530: expect (ObjNum 178 snap 40 seq_num 178) 2024-09-21T09:52:53.511 INFO:tasks.rados.rados.0.smithi033.stdout:531: rollback oid 27 current snap is 46 2024-09-21T09:52:53.511 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 27 to 45 2024-09-21T09:52:53.511 INFO:tasks.rados.rados.0.smithi033.stdout:523: finishing copy_from racing read to smithi03385152-13 2024-09-21T09:52:53.511 INFO:tasks.rados.rados.0.smithi033.stdout:527: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:52:53.512 INFO:tasks.rados.rados.0.smithi033.stdout:526: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:523: finishing copy_from to smithi03385152-13 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 138 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:529: finishing write tid 1 to smithi03385152-6 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:529: finishing write tid 2 to smithi03385152-6 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:529: finishing write tid 3 to smithi03385152-6 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:523: done (5 left) 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:526: done (4 left) 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:527: done (3 left) 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 46 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:531: finishing rollback tid 0 to smithi03385152-27 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:529: finishing write tid 4 to smithi03385152-6 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 156 (ObjNum 196 snap 46 seq_num 196) dirty exists 2024-09-21T09:52:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:529: left oid 6 (ObjNum 196 snap 46 seq_num 196) 2024-09-21T09:52:53.647 INFO:tasks.rados.rados.0.smithi033.stdout:531: finishing rollback tid 1 to smithi03385152-27 2024-09-21T09:52:53.647 INFO:tasks.rados.rados.0.smithi033.stdout:532: seq_num 197 ranges {0=507904,507904=401408} 2024-09-21T09:52:53.648 INFO:tasks.rados.rados.0.smithi033.stdout:532: writing smithi03385152-44 from 0 to 507904 tid 1 2024-09-21T09:52:53.650 INFO:tasks.rados.rados.0.smithi033.stdout:532: writing smithi03385152-44 from 507904 to 909312 tid 2 2024-09-21T09:52:53.650 INFO:tasks.rados.rados.0.smithi033.stdout:529: done (3 left) 2024-09-21T09:52:53.650 INFO:tasks.rados.rados.0.smithi033.stdout:533: snap_create 2024-09-21T09:52:53.650 INFO:tasks.rados.rados.0.smithi033.stdout:531: finishing rollback tid 2 to smithi03385152-27 2024-09-21T09:52:53.650 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 160 (ObjNum 191 snap 45 seq_num 191) dirty exists 2024-09-21T09:52:53.651 INFO:tasks.rados.rados.0.smithi033.stdout:530: expect (ObjNum 178 snap 40 seq_num 178) 2024-09-21T09:52:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: pgmap v12: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:53 smithi105 ceph-mon[31286]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:52:53.866 INFO:tasks.rados.rados.0.smithi033.stdout:532: finishing write tid 1 to smithi03385152-44 2024-09-21T09:52:53.866 INFO:tasks.rados.rados.0.smithi033.stdout:532: finishing write tid 2 to smithi03385152-44 2024-09-21T09:52:53.866 INFO:tasks.rados.rados.0.smithi033.stdout:532: finishing write tid 3 to smithi03385152-44 2024-09-21T09:52:53.867 INFO:tasks.rados.rados.0.smithi033.stdout:532: finishing write tid 4 to smithi03385152-44 2024-09-21T09:52:53.867 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 68 (ObjNum 197 snap 46 seq_num 197) dirty exists 2024-09-21T09:52:53.867 INFO:tasks.rados.rados.0.smithi033.stdout:532: left oid 44 (ObjNum 197 snap 46 seq_num 197) 2024-09-21T09:52:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: pgmap v12: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[39045]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: pgmap v12: 121 pgs: 121 active+clean; 114 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:53.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:53 smithi033 ceph-mon[27969]: osdmap e158: 8 total, 8 up, 8 in 2024-09-21T09:52:54.481 INFO:tasks.rados.rados.0.smithi033.stdout:530: done (3 left) 2024-09-21T09:52:54.481 INFO:tasks.rados.rados.0.smithi033.stdout:531: done (2 left) 2024-09-21T09:52:54.481 INFO:tasks.rados.rados.0.smithi033.stdout:532: done (1 left) 2024-09-21T09:52:54.481 INFO:tasks.rados.rados.0.smithi033.stdout:533: done (0 left) 2024-09-21T09:52:54.481 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 47 2024-09-21T09:52:54.481 INFO:tasks.rados.rados.0.smithi033.stdout:534: seq_num 198 ranges {5275648=524288,5799936=401408} 2024-09-21T09:52:54.512 INFO:tasks.rados.rados.0.smithi033.stdout:534: writing smithi03385152-18 from 5275648 to 5799936 tid 1 2024-09-21T09:52:54.514 INFO:tasks.rados.rados.0.smithi033.stdout:534: writing smithi03385152-18 from 5799936 to 6201344 tid 2 2024-09-21T09:52:54.515 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 47 2024-09-21T09:52:54.515 INFO:tasks.rados.rados.0.smithi033.stdout:535: seq_num 199 ranges {1310720=540672,1851392=401408} 2024-09-21T09:52:54.525 INFO:tasks.rados.rados.0.smithi033.stdout:535: writing smithi03385152-39 from 1310720 to 1851392 tid 1 2024-09-21T09:52:54.527 INFO:tasks.rados.rados.0.smithi033.stdout:535: writing smithi03385152-39 from 1851392 to 2252800 tid 2 2024-09-21T09:52:54.528 INFO:tasks.rados.rados.0.smithi033.stdout:536: setattr oid 49 current snap is 47 2024-09-21T09:52:54.528 INFO:tasks.rados.rados.0.smithi033.stdout:534: finishing write tid 1 to smithi03385152-18 2024-09-21T09:52:54.528 INFO:tasks.rados.rados.0.smithi033.stdout:534: finishing write tid 2 to smithi03385152-18 2024-09-21T09:52:54.528 INFO:tasks.rados.rados.0.smithi033.stdout:534: finishing write tid 3 to smithi03385152-18 2024-09-21T09:52:54.528 INFO:tasks.rados.rados.0.smithi033.stdout:534: finishing write tid 4 to smithi03385152-18 2024-09-21T09:52:54.528 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 113 (ObjNum 198 snap 47 seq_num 198) dirty exists 2024-09-21T09:52:54.528 INFO:tasks.rados.rados.0.smithi033.stdout:534: left oid 18 (ObjNum 198 snap 47 seq_num 198) 2024-09-21T09:52:54.529 INFO:tasks.rados.rados.0.smithi033.stdout:534: done (2 left) 2024-09-21T09:52:54.529 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 47 2024-09-21T09:52:54.529 INFO:tasks.rados.rados.0.smithi033.stdout:537: seq_num 200 ranges {4161536=557056,4718592=401408} 2024-09-21T09:52:54.555 INFO:tasks.rados.rados.0.smithi033.stdout:537: writing smithi03385152-8 from 4161536 to 4718592 tid 1 2024-09-21T09:52:54.558 INFO:tasks.rados.rados.0.smithi033.stdout:537: writing smithi03385152-8 from 4718592 to 5120000 tid 2 2024-09-21T09:52:54.558 INFO:tasks.rados.rados.0.smithi033.stdout:538: rollback oid 23 current snap is 47 2024-09-21T09:52:54.559 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 23 to 42 2024-09-21T09:52:54.559 INFO:tasks.rados.rados.0.smithi033.stdout:535: finishing write tid 1 to smithi03385152-39 2024-09-21T09:52:54.559 INFO:tasks.rados.rados.0.smithi033.stdout:535: finishing write tid 2 to smithi03385152-39 2024-09-21T09:52:54.559 INFO:tasks.rados.rados.0.smithi033.stdout:535: finishing write tid 3 to smithi03385152-39 2024-09-21T09:52:54.559 INFO:tasks.rados.rados.0.smithi033.stdout:535: finishing write tid 4 to smithi03385152-39 2024-09-21T09:52:54.560 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 61 (ObjNum 199 snap 47 seq_num 199) dirty exists 2024-09-21T09:52:54.560 INFO:tasks.rados.rados.0.smithi033.stdout:535: left oid 39 (ObjNum 199 snap 47 seq_num 199) 2024-09-21T09:52:54.560 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 228 (ObjNum 98 snap 17 seq_num 98) dirty exists 2024-09-21T09:52:54.560 INFO:tasks.rados.rados.0.smithi033.stdout:535: done (3 left) 2024-09-21T09:52:54.560 INFO:tasks.rados.rados.0.smithi033.stdout:536: done (2 left) 2024-09-21T09:52:54.560 INFO:tasks.rados.rados.0.smithi033.stdout:539: copy_from oid 46 from oid 12 current snap is 47 2024-09-21T09:52:54.560 INFO:tasks.rados.rados.0.smithi033.stdout:540: snap_remove snap 44 2024-09-21T09:52:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:54 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi033 to 2481M 2024-09-21T09:52:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:54 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:54 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:54 smithi105 ceph-mon[31286]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:52:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:54 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:54 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi033 to 2481M 2024-09-21T09:52:55.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[39045]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:52:55.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi033 to 2481M 2024-09-21T09:52:55.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[27969]: osdmap e159: 8 total, 8 up, 8 in 2024-09-21T09:52:55.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:54 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:55.484 INFO:tasks.rados.rados.0.smithi033.stdout:540: done (3 left) 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:541: copy_from oid 30 from oid 35 current snap is 47 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:537: finishing write tid 1 to smithi03385152-8 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:538: finishing rollback tid 0 to smithi03385152-23 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:538: finishing rollback tid 1 to smithi03385152-23 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:537: finishing write tid 2 to smithi03385152-8 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:538: finishing rollback tid 2 to smithi03385152-23 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 182 (ObjNum 180 snap 41 seq_num 180) dirty exists 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:537: finishing write tid 3 to smithi03385152-8 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:537: finishing write tid 4 to smithi03385152-8 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 142 (ObjNum 200 snap 47 seq_num 200) dirty exists 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:537: left oid 8 (ObjNum 200 snap 47 seq_num 200) 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:539: finishing copy_from racing read to smithi03385152-46 2024-09-21T09:52:55.485 INFO:tasks.rados.rados.0.smithi033.stdout:539: finishing copy_from to smithi03385152-46 2024-09-21T09:52:55.486 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 181 (ObjNum 177 snap 40 seq_num 177) dirty exists 2024-09-21T09:52:55.486 INFO:tasks.rados.rados.0.smithi033.stdout:537: done (3 left) 2024-09-21T09:52:55.486 INFO:tasks.rados.rados.0.smithi033.stdout:538: done (2 left) 2024-09-21T09:52:55.486 INFO:tasks.rados.rados.0.smithi033.stdout:539: done (1 left) 2024-09-21T09:52:55.486 INFO:tasks.rados.rados.0.smithi033.stdout:542: read oid 44 snap 43 2024-09-21T09:52:55.486 INFO:tasks.rados.rados.0.smithi033.stdout:542: expect (ObjNum 139 snap 32 seq_num 139) 2024-09-21T09:52:55.486 INFO:tasks.rados.rados.0.smithi033.stdout:543: copy_from oid 46 from oid 10 current snap is 47 2024-09-21T09:52:55.486 INFO:tasks.rados.rados.0.smithi033.stdout:544: delete oid 19 current snap is 47 2024-09-21T09:52:55.492 INFO:tasks.rados.rados.0.smithi033.stdout:543: finishing copy_from to smithi03385152-46 2024-09-21T09:52:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:543: got expected ENOENT (src dne) 2024-09-21T09:52:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:544: done (3 left) 2024-09-21T09:52:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:545: snap_create 2024-09-21T09:52:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:541: finishing copy_from racing read to smithi03385152-30 2024-09-21T09:52:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:543: finishing copy_from racing read to smithi03385152-46 2024-09-21T09:52:55.494 INFO:tasks.rados.rados.0.smithi033.stdout:541: finishing copy_from to smithi03385152-30 2024-09-21T09:52:55.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 256 (ObjNum 147 snap 34 seq_num 147) dirty exists 2024-09-21T09:52:55.504 INFO:tasks.rados.rados.0.smithi033.stdout:542: expect (ObjNum 139 snap 32 seq_num 139) 2024-09-21T09:52:55.940 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:55 smithi105 ceph-mon[31286]: pgmap v15: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 38 op/s 2024-09-21T09:52:55.940 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:55 smithi105 ceph-mon[31286]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:52:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:55 smithi033 ceph-mon[39045]: pgmap v15: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 38 op/s 2024-09-21T09:52:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:55 smithi033 ceph-mon[39045]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:52:56.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:55 smithi033 ceph-mon[27969]: pgmap v15: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 38 op/s 2024-09-21T09:52:56.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:55 smithi033 ceph-mon[27969]: osdmap e160: 8 total, 8 up, 8 in 2024-09-21T09:52:56.480 INFO:tasks.rados.rados.0.smithi033.stdout:541: done (3 left) 2024-09-21T09:52:56.480 INFO:tasks.rados.rados.0.smithi033.stdout:542: done (2 left) 2024-09-21T09:52:56.480 INFO:tasks.rados.rados.0.smithi033.stdout:543: done (1 left) 2024-09-21T09:52:56.480 INFO:tasks.rados.rados.0.smithi033.stdout:545: done (0 left) 2024-09-21T09:52:56.480 INFO:tasks.rados.rados.0.smithi033.stdout:546: read oid 45 snap -1 2024-09-21T09:52:56.481 INFO:tasks.rados.rados.0.smithi033.stdout:546: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:52:56.481 INFO:tasks.rados.rados.0.smithi033.stdout:547: setattr oid 1 current snap is 48 2024-09-21T09:52:56.482 INFO:tasks.rados.rados.0.smithi033.stdout:548: rmattr oid 34 current snap is 48 2024-09-21T09:52:56.482 INFO:tasks.rados.rados.0.smithi033.stdout:549: read oid 8 snap -1 2024-09-21T09:52:56.482 INFO:tasks.rados.rados.0.smithi033.stdout:549: expect (ObjNum 200 snap 47 seq_num 200) 2024-09-21T09:52:56.482 INFO:tasks.rados.rados.0.smithi033.stdout:550: snap_create 2024-09-21T09:52:56.487 INFO:tasks.rados.rados.0.smithi033.stdout:546: expect (ObjNum 13 snap 0 seq_num 13) 2024-09-21T09:52:56.498 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 267 (ObjNum 129 snap 0 seq_num 91210592) dirty exists 2024-09-21T09:52:56.498 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 143 (ObjNum 33 snap 0 seq_num 3) dirty exists 2024-09-21T09:52:56.505 INFO:tasks.rados.rados.0.smithi033.stdout:549: expect (ObjNum 200 snap 47 seq_num 200) 2024-09-21T09:52:57.482 INFO:tasks.rados.rados.0.smithi033.stdout:546: done (4 left) 2024-09-21T09:52:57.483 INFO:tasks.rados.rados.0.smithi033.stdout:547: done (3 left) 2024-09-21T09:52:57.483 INFO:tasks.rados.rados.0.smithi033.stdout:548: done (2 left) 2024-09-21T09:52:57.483 INFO:tasks.rados.rados.0.smithi033.stdout:549: done (1 left) 2024-09-21T09:52:57.483 INFO:tasks.rados.rados.0.smithi033.stdout:550: done (0 left) 2024-09-21T09:52:57.483 INFO:tasks.rados.rados.0.smithi033.stdout:551: read oid 3 snap -1 2024-09-21T09:52:57.483 INFO:tasks.rados.rados.0.smithi033.stdout:551: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:52:57.483 INFO:tasks.rados.rados.0.smithi033.stdout:552: setattr oid 25 current snap is 49 2024-09-21T09:52:57.484 INFO:tasks.rados.rados.0.smithi033.stdout:553: read oid 48 snap 45 2024-09-21T09:52:57.484 INFO:tasks.rados.rados.0.smithi033.stdout:553: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:52:57.484 INFO:tasks.rados.rados.0.smithi033.stdout:554: copy_from oid 26 from oid 12 current snap is 49 2024-09-21T09:52:57.484 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 49 2024-09-21T09:52:57.484 INFO:tasks.rados.rados.0.smithi033.stdout:555: seq_num 202 ranges {3612672=589824,4202496=401408} 2024-09-21T09:52:57.507 INFO:tasks.rados.rados.0.smithi033.stdout:555: writing smithi03385152-46 from 3612672 to 4202496 tid 1 2024-09-21T09:52:57.510 INFO:tasks.rados.rados.0.smithi033.stdout:555: writing smithi03385152-46 from 4202496 to 4603904 tid 2 2024-09-21T09:52:57.510 INFO:tasks.rados.rados.0.smithi033.stdout:556: rollback oid 34 current snap is 49 2024-09-21T09:52:57.510 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 42 2024-09-21T09:52:57.510 INFO:tasks.rados.rados.0.smithi033.stdout:551: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:52:57.530 INFO:tasks.rados.rados.0.smithi033.stdout:553: expect (ObjNum 41 snap 0 seq_num 41) 2024-09-21T09:52:57.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 47 (ObjNum 99 snap 17 seq_num 99) dirty exists 2024-09-21T09:52:57.549 INFO:tasks.rados.rados.0.smithi033.stdout:554: finishing copy_from racing read to smithi03385152-26 2024-09-21T09:52:57.549 INFO:tasks.rados.rados.0.smithi033.stdout:554: finishing copy_from to smithi03385152-26 2024-09-21T09:52:57.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 181 (ObjNum 177 snap 40 seq_num 177) dirty exists 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:555: finishing write tid 1 to smithi03385152-46 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:555: finishing write tid 2 to smithi03385152-46 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:551: done (5 left) 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:552: done (4 left) 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:553: done (3 left) 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:554: done (2 left) 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 49 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:555: finishing write tid 3 to smithi03385152-46 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:556: finishing rollback tid 0 to smithi03385152-34 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:556: finishing rollback tid 1 to smithi03385152-34 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:556: finishing rollback tid 2 to smithi03385152-34 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 275 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:555: finishing write tid 4 to smithi03385152-46 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 271 (ObjNum 202 snap 49 seq_num 202) dirty exists 2024-09-21T09:52:57.550 INFO:tasks.rados.rados.0.smithi033.stdout:555: left oid 46 (ObjNum 202 snap 49 seq_num 202) 2024-09-21T09:52:57.551 INFO:tasks.rados.rados.0.smithi033.stdout:557: seq_num 203 ranges {0=606208,606208=401408} 2024-09-21T09:52:57.552 INFO:tasks.rados.rados.0.smithi033.stdout:557: writing smithi03385152-36 from 0 to 606208 tid 1 2024-09-21T09:52:57.555 INFO:tasks.rados.rados.0.smithi033.stdout:557: writing smithi03385152-36 from 606208 to 1007616 tid 2 2024-09-21T09:52:57.555 INFO:tasks.rados.rados.0.smithi033.stdout:555: done (2 left) 2024-09-21T09:52:57.555 INFO:tasks.rados.rados.0.smithi033.stdout:556: done (1 left) 2024-09-21T09:52:57.556 INFO:tasks.rados.rados.0.smithi033.stdout:558: rollback oid 10 current snap is 49 2024-09-21T09:52:57.556 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 45 2024-09-21T09:52:57.556 INFO:tasks.rados.rados.0.smithi033.stdout:559: rollback oid 4 current snap is 49 2024-09-21T09:52:57.556 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 4 to 39 2024-09-21T09:52:57.556 INFO:tasks.rados.rados.0.smithi033.stdout:560: read oid 19 snap 41 2024-09-21T09:52:57.556 INFO:tasks.rados.rados.0.smithi033.stdout:560: expect deleted 2024-09-21T09:52:57.556 INFO:tasks.rados.rados.0.smithi033.stdout:561: copy_from oid 17 from oid 44 current snap is 49 2024-09-21T09:52:57.556 INFO:tasks.rados.rados.0.smithi033.stdout:562: snap_remove snap 48 2024-09-21T09:52:57.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:57 smithi105 ceph-mon[31286]: pgmap v18: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 38 op/s 2024-09-21T09:52:57.639 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:57 smithi105 ceph-mon[31286]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:52:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:57 smithi033 ceph-mon[39045]: pgmap v18: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 38 op/s 2024-09-21T09:52:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:57 smithi033 ceph-mon[39045]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:52:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:57 smithi033 ceph-mon[27969]: pgmap v18: 121 pgs: 121 active+clean; 121 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.8 MiB/s wr, 38 op/s 2024-09-21T09:52:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:57 smithi033 ceph-mon[27969]: osdmap e161: 8 total, 8 up, 8 in 2024-09-21T09:52:58.494 INFO:tasks.rados.rados.0.smithi033.stdout:562: done (5 left) 2024-09-21T09:52:58.494 INFO:tasks.rados.rados.0.smithi033.stdout:563: snap_remove snap 43 2024-09-21T09:52:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:52:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:52:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: Adjusting osd_memory_target on smithi105 to 2480M 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:52:58.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: Adjusting osd_memory_target on smithi105 to 2480M 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:58.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.750 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.751 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.752 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: osdmap e162: 8 total, 8 up, 8 in 2024-09-21T09:52:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:52:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: Adjusting osd_memory_target on smithi105 to 2480M 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:52:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:52:58.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:58.784 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:58 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:52:59.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:52:58 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:52:58] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:52:59.497 INFO:tasks.rados.rados.0.smithi033.stdout:563: done (5 left) 2024-09-21T09:52:59.497 INFO:tasks.rados.rados.0.smithi033.stdout:564: rollback oid 21 current snap is 49 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 30 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:565: delete oid 31 current snap is 49 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:558: finishing rollback tid 1 to smithi03385152-10 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 59 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:559: finishing rollback tid 1 to smithi03385152-4 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 43 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:557: finishing write tid 1 to smithi03385152-36 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:557: finishing write tid 2 to smithi03385152-36 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:557: finishing write tid 3 to smithi03385152-36 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:557: finishing write tid 4 to smithi03385152-36 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 343 (ObjNum 203 snap 49 seq_num 203) dirty exists 2024-09-21T09:52:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:557: left oid 36 (ObjNum 203 snap 49 seq_num 203) 2024-09-21T09:52:59.499 INFO:tasks.rados.rados.0.smithi033.stdout:561: finishing copy_from racing read to smithi03385152-17 2024-09-21T09:52:59.499 INFO:tasks.rados.rados.0.smithi033.stdout:561: finishing copy_from to smithi03385152-17 2024-09-21T09:52:59.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 68 (ObjNum 197 snap 46 seq_num 197) dirty exists 2024-09-21T09:52:59.512 INFO:tasks.rados.rados.0.smithi033.stdout:564: finishing rollback tid 0 to smithi03385152-21 2024-09-21T09:52:59.512 INFO:tasks.rados.rados.0.smithi033.stdout:564: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:52:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:564: finishing rollback tid 2 to smithi03385152-21 2024-09-21T09:52:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 188 (ObjNum 112 snap 25 seq_num 112) dirty exists 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:557: done (6 left) 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:558: done (5 left) 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:559: done (4 left) 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:560: done (3 left) 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:561: done (2 left) 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:564: done (1 left) 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:565: done (0 left) 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:566: read oid 42 snap -1 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:566: expect deleted 2024-09-21T09:52:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:567: setattr oid 5 current snap is 49 2024-09-21T09:52:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:566: done (1 left) 2024-09-21T09:52:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:568: rmattr oid 24 current snap is 49 2024-09-21T09:52:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:568: done (1 left) 2024-09-21T09:52:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 49 2024-09-21T09:52:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:569: seq_num 205 ranges {778240=638976,1417216=401408} 2024-09-21T09:52:59.524 INFO:tasks.rados.rados.0.smithi033.stdout:569: writing smithi03385152-3 from 778240 to 1417216 tid 1 2024-09-21T09:52:59.526 INFO:tasks.rados.rados.0.smithi033.stdout:569: writing smithi03385152-3 from 1417216 to 1818624 tid 2 2024-09-21T09:52:59.526 INFO:tasks.rados.rados.0.smithi033.stdout:570: snap_create 2024-09-21T09:52:59.526 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 66 (ObjNum 33 snap 0 seq_num 90775344) dirty exists 2024-09-21T09:52:59.529 INFO:tasks.rados.rados.0.smithi033.stdout:569: finishing write tid 1 to smithi03385152-3 2024-09-21T09:52:59.529 INFO:tasks.rados.rados.0.smithi033.stdout:569: finishing write tid 2 to smithi03385152-3 2024-09-21T09:52:59.529 INFO:tasks.rados.rados.0.smithi033.stdout:569: finishing write tid 3 to smithi03385152-3 2024-09-21T09:52:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:569: finishing write tid 4 to smithi03385152-3 2024-09-21T09:52:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 289 (ObjNum 205 snap 49 seq_num 205) dirty exists 2024-09-21T09:52:59.530 INFO:tasks.rados.rados.0.smithi033.stdout:569: left oid 3 (ObjNum 205 snap 49 seq_num 205) 2024-09-21T09:52:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:59 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:52:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:59 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:52:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:59 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:52:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:59 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:52:59.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:59 smithi105 ceph-mon[31286]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:52:59.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:59 smithi105 ceph-mon[31286]: Deploying daemon alertmanager.a on smithi033 2024-09-21T09:52:59.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:59 smithi105 ceph-mon[31286]: pgmap v21: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-21T09:52:59.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:52:59 smithi105 ceph-mon[31286]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:52:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:52:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:52:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:52:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:52:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[39045]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:52:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[39045]: Deploying daemon alertmanager.a on smithi033 2024-09-21T09:52:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[39045]: pgmap v21: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-21T09:52:59.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[39045]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:52:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:52:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:52:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:52:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:52:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[27969]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-21T09:52:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[27969]: Deploying daemon alertmanager.a on smithi033 2024-09-21T09:52:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[27969]: pgmap v21: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-21T09:52:59.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:52:59 smithi033 ceph-mon[27969]: osdmap e163: 8 total, 8 up, 8 in 2024-09-21T09:53:00.507 INFO:tasks.rados.rados.0.smithi033.stdout:567: done (2 left) 2024-09-21T09:53:00.508 INFO:tasks.rados.rados.0.smithi033.stdout:569: done (1 left) 2024-09-21T09:53:00.508 INFO:tasks.rados.rados.0.smithi033.stdout:570: done (0 left) 2024-09-21T09:53:00.508 INFO:tasks.rados.rados.0.smithi033.stdout:571: snap_remove snap 47 2024-09-21T09:53:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:00 smithi105 ceph-mon[31286]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:53:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:00 smithi105 ceph-mon[31286]: mgrmap e28: x(active, since 13s), standbys: y 2024-09-21T09:53:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:00 smithi033 ceph-mon[39045]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:53:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:00 smithi033 ceph-mon[39045]: mgrmap e28: x(active, since 13s), standbys: y 2024-09-21T09:53:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:00 smithi033 ceph-mon[27969]: osdmap e164: 8 total, 8 up, 8 in 2024-09-21T09:53:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:00 smithi033 ceph-mon[27969]: mgrmap e28: x(active, since 13s), standbys: y 2024-09-21T09:53:01.481 INFO:tasks.rados.rados.0.smithi033.stdout:571: done (0 left) 2024-09-21T09:53:01.481 INFO:tasks.rados.rados.0.smithi033.stdout:572: snap_remove snap 49 2024-09-21T09:53:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:01 smithi105 ceph-mon[31286]: pgmap v24: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-21T09:53:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:01 smithi105 ceph-mon[31286]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:53:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:01 smithi105 ceph-mon[31286]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:53:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:01 smithi033 ceph-mon[39045]: pgmap v24: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-21T09:53:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:01 smithi033 ceph-mon[39045]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:53:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:01 smithi033 ceph-mon[39045]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:53:01.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:01 smithi033 ceph-mon[27969]: pgmap v24: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-21T09:53:01.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:01 smithi033 ceph-mon[27969]: osdmap e165: 8 total, 8 up, 8 in 2024-09-21T09:53:01.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:01 smithi033 ceph-mon[27969]: osdmap e166: 8 total, 8 up, 8 in 2024-09-21T09:53:02.485 INFO:tasks.rados.rados.0.smithi033.stdout:572: done (0 left) 2024-09-21T09:53:02.485 INFO:tasks.rados.rados.0.smithi033.stdout:573: rollback oid 12 current snap is 50 2024-09-21T09:53:02.485 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 12 to 40 2024-09-21T09:53:02.485 INFO:tasks.rados.rados.0.smithi033.stdout:574: read oid 21 snap -1 2024-09-21T09:53:02.485 INFO:tasks.rados.rados.0.smithi033.stdout:574: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:53:02.485 INFO:tasks.rados.rados.0.smithi033.stdout:575: rmattr oid 8 current snap is 50 2024-09-21T09:53:02.486 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 50 2024-09-21T09:53:02.486 INFO:tasks.rados.rados.0.smithi033.stdout:576: seq_num 206 ranges {696320=655360,1351680=401408} 2024-09-21T09:53:02.493 INFO:tasks.rados.rados.0.smithi033.stdout:576: writing smithi03385152-48 from 696320 to 1351680 tid 1 2024-09-21T09:53:02.497 INFO:tasks.rados.rados.0.smithi033.stdout:576: writing smithi03385152-48 from 1351680 to 1753088 tid 2 2024-09-21T09:53:02.497 INFO:tasks.rados.rados.0.smithi033.stdout:577: rollback oid 25 current snap is 50 2024-09-21T09:53:02.497 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 25 to 30 2024-09-21T09:53:02.497 INFO:tasks.rados.rados.0.smithi033.stdout:573: finishing rollback tid 0 to smithi03385152-12 2024-09-21T09:53:02.497 INFO:tasks.rados.rados.0.smithi033.stdout:573: finishing rollback tid 1 to smithi03385152-12 2024-09-21T09:53:02.497 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 144 (ObjNum 200 snap 47 seq_num 200) dirty exists 2024-09-21T09:53:02.497 INFO:tasks.rados.rados.0.smithi033.stdout:573: finishing rollback tid 2 to smithi03385152-12 2024-09-21T09:53:02.497 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 231 (ObjNum 177 snap 40 seq_num 177) dirty exists 2024-09-21T09:53:02.498 INFO:tasks.rados.rados.0.smithi033.stdout:573: done (4 left) 2024-09-21T09:53:02.498 INFO:tasks.rados.rados.0.smithi033.stdout:575: done (3 left) 2024-09-21T09:53:02.498 INFO:tasks.rados.rados.0.smithi033.stdout:578: read oid 29 snap -1 2024-09-21T09:53:02.498 INFO:tasks.rados.rados.0.smithi033.stdout:578: expect deleted 2024-09-21T09:53:02.498 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 50 2024-09-21T09:53:02.498 INFO:tasks.rados.rados.0.smithi033.stdout:579: seq_num 207 ranges {786432=671744,1458176=401408} 2024-09-21T09:53:02.505 INFO:tasks.rados.rados.0.smithi033.stdout:579: writing smithi03385152-33 from 786432 to 1458176 tid 1 2024-09-21T09:53:02.508 INFO:tasks.rados.rados.0.smithi033.stdout:579: writing smithi03385152-33 from 1458176 to 1859584 tid 2 2024-09-21T09:53:02.509 INFO:tasks.rados.rados.0.smithi033.stdout:580: snap_create 2024-09-21T09:53:02.509 INFO:tasks.rados.rados.0.smithi033.stdout:576: finishing write tid 1 to smithi03385152-48 2024-09-21T09:53:02.509 INFO:tasks.rados.rados.0.smithi033.stdout:576: finishing write tid 2 to smithi03385152-48 2024-09-21T09:53:02.509 INFO:tasks.rados.rados.0.smithi033.stdout:576: finishing write tid 3 to smithi03385152-48 2024-09-21T09:53:02.509 INFO:tasks.rados.rados.0.smithi033.stdout:577: finishing rollback tid 0 to smithi03385152-25 2024-09-21T09:53:02.509 INFO:tasks.rados.rados.0.smithi033.stdout:577: finishing rollback tid 1 to smithi03385152-25 2024-09-21T09:53:02.509 INFO:tasks.rados.rados.0.smithi033.stdout:577: finishing rollback tid 2 to smithi03385152-25 2024-09-21T09:53:02.509 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 56 (ObjNum 99 snap 17 seq_num 99) dirty exists 2024-09-21T09:53:02.510 INFO:tasks.rados.rados.0.smithi033.stdout:576: finishing write tid 4 to smithi03385152-48 2024-09-21T09:53:02.510 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 299 (ObjNum 206 snap 50 seq_num 206) dirty exists 2024-09-21T09:53:02.510 INFO:tasks.rados.rados.0.smithi033.stdout:576: left oid 48 (ObjNum 206 snap 50 seq_num 206) 2024-09-21T09:53:02.510 INFO:tasks.rados.rados.0.smithi033.stdout:574: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:53:02.640 INFO:tasks.rados.rados.0.smithi033.stdout:579: finishing write tid 1 to smithi03385152-33 2024-09-21T09:53:02.640 INFO:tasks.rados.rados.0.smithi033.stdout:579: finishing write tid 2 to smithi03385152-33 2024-09-21T09:53:02.641 INFO:tasks.rados.rados.0.smithi033.stdout:579: finishing write tid 3 to smithi03385152-33 2024-09-21T09:53:02.641 INFO:tasks.rados.rados.0.smithi033.stdout:579: finishing write tid 4 to smithi03385152-33 2024-09-21T09:53:02.641 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 172 (ObjNum 207 snap 50 seq_num 207) dirty exists 2024-09-21T09:53:02.641 INFO:tasks.rados.rados.0.smithi033.stdout:579: left oid 33 (ObjNum 207 snap 50 seq_num 207) 2024-09-21T09:53:02.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:02 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:53:02] "GET /metrics HTTP/1.1" 200 35928 "" "Prometheus/2.43.0" 2024-09-21T09:53:03.168 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:53:03.514 INFO:tasks.rados.rados.0.smithi033.stdout:574: done (5 left) 2024-09-21T09:53:03.514 INFO:tasks.rados.rados.0.smithi033.stdout:576: done (4 left) 2024-09-21T09:53:03.514 INFO:tasks.rados.rados.0.smithi033.stdout:577: done (3 left) 2024-09-21T09:53:03.514 INFO:tasks.rados.rados.0.smithi033.stdout:578: done (2 left) 2024-09-21T09:53:03.514 INFO:tasks.rados.rados.0.smithi033.stdout:579: done (1 left) 2024-09-21T09:53:03.514 INFO:tasks.rados.rados.0.smithi033.stdout:580: done (0 left) 2024-09-21T09:53:03.514 INFO:tasks.rados.rados.0.smithi033.stdout:581: delete oid 2 current snap is 51 2024-09-21T09:53:03.516 INFO:tasks.rados.rados.0.smithi033.stdout:581: done (0 left) 2024-09-21T09:53:03.517 INFO:tasks.rados.rados.0.smithi033.stdout:582: delete oid 5 current snap is 51 2024-09-21T09:53:03.519 INFO:tasks.rados.rados.0.smithi033.stdout:582: done (0 left) 2024-09-21T09:53:03.519 INFO:tasks.rados.rados.0.smithi033.stdout:583: read oid 43 snap -1 2024-09-21T09:53:03.520 INFO:tasks.rados.rados.0.smithi033.stdout:583: expect deleted 2024-09-21T09:53:03.520 INFO:tasks.rados.rados.0.smithi033.stdout:584: read oid 28 snap 42 2024-09-21T09:53:03.520 INFO:tasks.rados.rados.0.smithi033.stdout:584: expect (ObjNum 152 snap 34 seq_num 152) 2024-09-21T09:53:03.520 INFO:tasks.rados.rados.0.smithi033.stdout:585: delete oid 17 current snap is 51 2024-09-21T09:53:03.522 INFO:tasks.rados.rados.0.smithi033.stdout:583: done (2 left) 2024-09-21T09:53:03.522 INFO:tasks.rados.rados.0.smithi033.stdout:585: done (1 left) 2024-09-21T09:53:03.522 INFO:tasks.rados.rados.0.smithi033.stdout:586: copy_from oid 45 from oid 13 current snap is 51 2024-09-21T09:53:03.522 INFO:tasks.rados.rados.0.smithi033.stdout:587: rmattr oid 36 current snap is 51 2024-09-21T09:53:03.522 INFO:tasks.rados.rados.0.smithi033.stdout:587: done (2 left) 2024-09-21T09:53:03.523 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 51 2024-09-21T09:53:03.523 INFO:tasks.rados.rados.0.smithi033.stdout:588: seq_num 211 ranges {24576=737280,761856=401408} 2024-09-21T09:53:03.526 INFO:tasks.rados.rados.0.smithi033.stdout:588: writing smithi03385152-11 from 24576 to 761856 tid 1 2024-09-21T09:53:03.529 INFO:tasks.rados.rados.0.smithi033.stdout:588: writing smithi03385152-11 from 761856 to 1163264 tid 2 2024-09-21T09:53:03.529 INFO:tasks.rados.rados.0.smithi033.stdout:589: read oid 19 snap -1 2024-09-21T09:53:03.530 INFO:tasks.rados.rados.0.smithi033.stdout:589: expect deleted 2024-09-21T09:53:03.530 INFO:tasks.rados.rados.0.smithi033.stdout:590: read oid 49 snap 29 2024-09-21T09:53:03.530 INFO:tasks.rados.rados.0.smithi033.stdout:590: expect deleted 2024-09-21T09:53:03.530 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 51 2024-09-21T09:53:03.530 INFO:tasks.rados.rados.0.smithi033.stdout:591: seq_num 212 ranges {778240=753664,1531904=401408} 2024-09-21T09:53:03.538 INFO:tasks.rados.rados.0.smithi033.stdout:591: writing smithi03385152-22 from 778240 to 1531904 tid 1 2024-09-21T09:53:03.542 INFO:tasks.rados.rados.0.smithi033.stdout:591: writing smithi03385152-22 from 1531904 to 1933312 tid 2 2024-09-21T09:53:03.542 INFO:tasks.rados.rados.0.smithi033.stdout:592: copy_from oid 42 from oid 24 current snap is 51 2024-09-21T09:53:03.542 INFO:tasks.rados.rados.0.smithi033.stdout:588: finishing write tid 1 to smithi03385152-11 2024-09-21T09:53:03.542 INFO:tasks.rados.rados.0.smithi033.stdout:588: finishing write tid 2 to smithi03385152-11 2024-09-21T09:53:03.542 INFO:tasks.rados.rados.0.smithi033.stdout:588: finishing write tid 3 to smithi03385152-11 2024-09-21T09:53:03.542 INFO:tasks.rados.rados.0.smithi033.stdout:584: expect (ObjNum 152 snap 34 seq_num 152) 2024-09-21T09:53:03.643 INFO:tasks.rados.rados.0.smithi033.stdout:588: finishing write tid 4 to smithi03385152-11 2024-09-21T09:53:03.643 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 149 (ObjNum 211 snap 51 seq_num 211) dirty exists 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:588: left oid 11 (ObjNum 211 snap 51 seq_num 211) 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:586: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:584: done (6 left) 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:588: done (5 left) 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:589: done (4 left) 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:590: done (3 left) 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 51 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:586: finishing copy_from to smithi03385152-45 2024-09-21T09:53:03.644 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 138 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:53:03.645 INFO:tasks.rados.rados.0.smithi033.stdout:591: finishing write tid 1 to smithi03385152-22 2024-09-21T09:53:03.645 INFO:tasks.rados.rados.0.smithi033.stdout:591: finishing write tid 2 to smithi03385152-22 2024-09-21T09:53:03.645 INFO:tasks.rados.rados.0.smithi033.stdout:591: finishing write tid 3 to smithi03385152-22 2024-09-21T09:53:03.645 INFO:tasks.rados.rados.0.smithi033.stdout:592: finishing copy_from racing read to smithi03385152-42 2024-09-21T09:53:03.645 INFO:tasks.rados.rados.0.smithi033.stdout:592: finishing copy_from to smithi03385152-42 2024-09-21T09:53:03.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 16 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-21T09:53:03.645 INFO:tasks.rados.rados.0.smithi033.stdout:591: finishing write tid 4 to smithi03385152-22 2024-09-21T09:53:03.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 305 (ObjNum 212 snap 51 seq_num 212) dirty exists 2024-09-21T09:53:03.646 INFO:tasks.rados.rados.0.smithi033.stdout:591: left oid 22 (ObjNum 212 snap 51 seq_num 212) 2024-09-21T09:53:03.646 INFO:tasks.rados.rados.0.smithi033.stdout:593: seq_num 213 ranges {0=770048,770048=401408} 2024-09-21T09:53:03.647 INFO:tasks.rados.rados.0.smithi033.stdout:593: writing smithi03385152-1 from 0 to 770048 tid 1 2024-09-21T09:53:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:593: writing smithi03385152-1 from 770048 to 1171456 tid 2 2024-09-21T09:53:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:586: done (3 left) 2024-09-21T09:53:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:591: done (2 left) 2024-09-21T09:53:03.651 INFO:tasks.rados.rados.0.smithi033.stdout:592: done (1 left) 2024-09-21T09:53:03.652 INFO:tasks.rados.rados.0.smithi033.stdout:594: rollback oid 26 current snap is 51 2024-09-21T09:53:03.652 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 26 to 42 2024-09-21T09:53:03.652 INFO:tasks.rados.rados.0.smithi033.stdout:595: read oid 33 snap -1 2024-09-21T09:53:03.652 INFO:tasks.rados.rados.0.smithi033.stdout:595: expect (ObjNum 207 snap 50 seq_num 207) 2024-09-21T09:53:03.652 INFO:tasks.rados.rados.0.smithi033.stdout:596: read oid 11 snap -1 2024-09-21T09:53:03.653 INFO:tasks.rados.rados.0.smithi033.stdout:596: expect (ObjNum 211 snap 51 seq_num 211) 2024-09-21T09:53:03.653 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 51 2024-09-21T09:53:03.653 INFO:tasks.rados.rados.0.smithi033.stdout:597: seq_num 214 ranges {0=794624,794624=401408} 2024-09-21T09:53:03.655 INFO:tasks.rados.rados.0.smithi033.stdout:597: writing smithi03385152-29 from 0 to 794624 tid 1 2024-09-21T09:53:03.658 INFO:tasks.rados.rados.0.smithi033.stdout:597: writing smithi03385152-29 from 794624 to 1196032 tid 2 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:598: read oid 5 snap -1 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:598: expect deleted 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:599: rollback oid 8 current snap is 51 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 8 to 29 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:594: finishing rollback tid 0 to smithi03385152-26 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:594: finishing rollback tid 1 to smithi03385152-26 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:593: finishing write tid 1 to smithi03385152-1 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:594: finishing rollback tid 2 to smithi03385152-26 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 235 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:593: finishing write tid 2 to smithi03385152-1 2024-09-21T09:53:03.659 INFO:tasks.rados.rados.0.smithi033.stdout:596: expect (ObjNum 211 snap 51 seq_num 211) 2024-09-21T09:53:03.691 INFO:tasks.rados.rados.0.smithi033.stdout:593: finishing write tid 3 to smithi03385152-1 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:593: finishing write tid 4 to smithi03385152-1 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 147 (ObjNum 213 snap 51 seq_num 213) dirty exists 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:593: left oid 1 (ObjNum 213 snap 51 seq_num 213) 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:593: done (6 left) 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:594: done (5 left) 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:596: done (4 left) 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:600: setattr oid 19 current snap is 51 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:599: finishing rollback tid 0 to smithi03385152-8 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:597: finishing write tid 1 to smithi03385152-29 2024-09-21T09:53:03.692 INFO:tasks.rados.rados.0.smithi033.stdout:599: finishing rollback tid 1 to smithi03385152-8 2024-09-21T09:53:03.693 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 149 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T09:53:03.693 INFO:tasks.rados.rados.0.smithi033.stdout:595: expect (ObjNum 207 snap 50 seq_num 207) 2024-09-21T09:53:03.696 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:03 smithi033 ceph-mon[39045]: pgmap v27: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:03.697 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:03 smithi033 ceph-mon[39045]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:53:03.697 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:03 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:03.697 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:03 smithi033 ceph-mon[27969]: pgmap v27: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:03.697 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:03 smithi033 ceph-mon[27969]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:53:03.698 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:03 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:03.759 INFO:tasks.rados.rados.0.smithi033.stdout:597: finishing write tid 2 to smithi03385152-29 2024-09-21T09:53:03.759 INFO:tasks.rados.rados.0.smithi033.stdout:597: finishing write tid 3 to smithi03385152-29 2024-09-21T09:53:03.759 INFO:tasks.rados.rados.0.smithi033.stdout:597: finishing write tid 4 to smithi03385152-29 2024-09-21T09:53:03.760 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 354 (ObjNum 214 snap 51 seq_num 214) dirty exists 2024-09-21T09:53:03.760 INFO:tasks.rados.rados.0.smithi033.stdout:597: left oid 29 (ObjNum 214 snap 51 seq_num 214) 2024-09-21T09:53:03.760 INFO:tasks.rados.rados.0.smithi033.stdout:595: done (4 left) 2024-09-21T09:53:03.760 INFO:tasks.rados.rados.0.smithi033.stdout:597: done (3 left) 2024-09-21T09:53:03.760 INFO:tasks.rados.rados.0.smithi033.stdout:598: done (2 left) 2024-09-21T09:53:03.760 INFO:tasks.rados.rados.0.smithi033.stdout:599: done (1 left) 2024-09-21T09:53:03.760 INFO:tasks.rados.rados.0.smithi033.stdout:601: delete oid 28 current snap is 51 2024-09-21T09:53:03.761 INFO:tasks.rados.rados.0.smithi033.stdout:601: done (1 left) 2024-09-21T09:53:03.761 INFO:tasks.rados.rados.0.smithi033.stdout:602: rollback oid 31 current snap is 51 2024-09-21T09:53:03.761 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 83 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-21T09:53:03.762 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 31 to 29 2024-09-21T09:53:03.762 INFO:tasks.rados.rados.0.smithi033.stdout:600: done (1 left) 2024-09-21T09:53:03.762 INFO:tasks.rados.rados.0.smithi033.stdout:603: setattr oid 42 current snap is 51 2024-09-21T09:53:03.763 INFO:tasks.rados.rados.0.smithi033.stdout:602: finishing rollback tid 1 to smithi03385152-31 2024-09-21T09:53:03.763 INFO:tasks.rados.rados.0.smithi033.stdout:602: finishing rollback tid 2 to smithi03385152-31 2024-09-21T09:53:03.763 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 356 (ObjNum 121 snap 28 seq_num 121) dirty exists 2024-09-21T09:53:03.764 INFO:tasks.rados.rados.0.smithi033.stdout:602: done (1 left) 2024-09-21T09:53:03.764 INFO:tasks.rados.rados.0.smithi033.stdout:604: setattr oid 6 current snap is 51 2024-09-21T09:53:03.765 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 176 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-21T09:53:03.766 INFO:tasks.rados.rados.0.smithi033.stdout:603: done (1 left) 2024-09-21T09:53:03.766 INFO:tasks.rados.rados.0.smithi033.stdout:605: snap_remove snap 46 2024-09-21T09:53:03.778 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:53:03.778 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (3m) 11s ago 4m 26.6M - 0.25.0 c8568f914cd2 f5c871f25675 2024-09-21T09:53:03.778 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (4m) 9s ago 4m 85.7M - 9.4.7 954c08fa6188 39f7f9958856 2024-09-21T09:53:03.778 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (5m) 11s ago 5m 14.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (5m) 9s ago 5m 18.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283 running (36s) 9s ago 9m 497M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b10e03ec7c52 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:9283,8765 running (11m) 11s ago 10m 301M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 21061eedc154 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (11m) 11s ago 11m 56.0M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 220cb0bc67bb 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (9m) 9s ago 9m 46.1M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c2741af3f853 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (9m) 11s ago 9m 45.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f8b68f40cc5 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (3m) 11s ago 3m 15.5M - 1.5.0 0da6a335fe13 ae258c372f47 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (3m) 9s ago 3m 16.9M - 1.5.0 0da6a335fe13 446328798f33 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (9m) 11s ago 8m 98.4M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (8m) 11s ago 8m 105M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (8m) 11s ago 8m 83.7M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (7m) 11s ago 7m 68.3M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (7m) 9s ago 7m 98.8M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (7m) 9s ago 7m 100M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:53:03.779 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (6m) 9s ago 6m 66.8M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:53:03.780 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (6m) 9s ago 6m 78.4M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:53:03.780 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (3m) 9s ago 4m 56.6M - 2.43.0 a07b618ecd1d 9b3306609c60 2024-09-21T09:53:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:03 smithi105 ceph-mon[31286]: pgmap v27: 121 pgs: 121 active+clean; 139 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:03 smithi105 ceph-mon[31286]: osdmap e167: 8 total, 8 up, 8 in 2024-09-21T09:53:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:03 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 1, 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:53:04.131 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:53:04.132 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:53:04.132 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:53:04.132 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:53:04.132 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 14, 2024-09-21T09:53:04.132 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:53:04.132 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:53:04.132 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:53:04.430 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:53:04.430 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:53:04.430 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:53:04.431 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:53:04.431 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-21T09:53:04.431 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "1/20 daemons upgraded", 2024-09-21T09:53:04.431 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "", 2024-09-21T09:53:04.431 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:53:04.431 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:53:04.519 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 174 (ObjNum 196 snap 46 seq_num 196) dirty exists 2024-09-21T09:53:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:604: done (1 left) 2024-09-21T09:53:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:605: done (0 left) 2024-09-21T09:53:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:606: delete oid 36 current snap is 51 2024-09-21T09:53:04.530 INFO:tasks.rados.rados.0.smithi033.stdout:606: done (0 left) 2024-09-21T09:53:04.530 INFO:tasks.rados.rados.0.smithi033.stdout:607: read oid 42 snap -1 2024-09-21T09:53:04.530 INFO:tasks.rados.rados.0.smithi033.stdout:607: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:53:04.530 INFO:tasks.rados.rados.0.smithi033.stdout:608: read oid 26 snap -1 2024-09-21T09:53:04.530 INFO:tasks.rados.rados.0.smithi033.stdout:608: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:53:04.530 INFO:tasks.rados.rados.0.smithi033.stdout:609: setattr oid 7 current snap is 51 2024-09-21T09:53:04.531 INFO:tasks.rados.rados.0.smithi033.stdout:610: delete oid 43 current snap is 51 2024-09-21T09:53:04.533 INFO:tasks.rados.rados.0.smithi033.stdout:610: done (3 left) 2024-09-21T09:53:04.533 INFO:tasks.rados.rados.0.smithi033.stdout:611: snap_create 2024-09-21T09:53:04.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 362 (ObjNum 129 snap 0 seq_num 90882888) dirty exists 2024-09-21T09:53:04.536 INFO:tasks.rados.rados.0.smithi033.stdout:607: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:53:04.579 INFO:tasks.rados.rados.0.smithi033.stdout:608: expect (ObjNum 46 snap 0 seq_num 46) 2024-09-21T09:53:04.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[39045]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[39045]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[39045]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:53:04.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[39045]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2756538720' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[27969]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[27969]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[27969]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:53:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[27969]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:04 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2756538720' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:04 smithi105 ceph-mon[31286]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:04 smithi105 ceph-mon[31286]: from='client.25007 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:04 smithi105 ceph-mon[31286]: osdmap e168: 8 total, 8 up, 8 in 2024-09-21T09:53:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:04 smithi105 ceph-mon[31286]: from='client.15105 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:04.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:04 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2756538720' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:05.532 INFO:tasks.rados.rados.0.smithi033.stdout:607: done (3 left) 2024-09-21T09:53:05.532 INFO:tasks.rados.rados.0.smithi033.stdout:608: done (2 left) 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:609: done (1 left) 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:611: done (0 left) 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:612: rollback oid 47 current snap is 52 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 29 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:613: read oid 10 snap -1 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:613: expect deleted 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:614: read oid 42 snap 41 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:614: expect deleted 2024-09-21T09:53:05.533 INFO:tasks.rados.rados.0.smithi033.stdout:615: snap_remove snap 39 2024-09-21T09:53:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:05 smithi105 ceph-mon[31286]: from='client.25025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:05 smithi105 ceph-mon[31286]: pgmap v30: 121 pgs: 121 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-21T09:53:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:05 smithi105 ceph-mon[31286]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:53:05.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:05 smithi033 ceph-mon[27969]: from='client.25025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:05.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:05 smithi033 ceph-mon[27969]: pgmap v30: 121 pgs: 121 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-21T09:53:05.986 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:05 smithi033 ceph-mon[27969]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:53:05.986 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:05 smithi033 ceph-mon[39045]: from='client.25025 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:05.986 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:05 smithi033 ceph-mon[39045]: pgmap v30: 121 pgs: 121 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-21T09:53:05.987 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:05 smithi033 ceph-mon[39045]: osdmap e169: 8 total, 8 up, 8 in 2024-09-21T09:53:06.489 INFO:tasks.rados.rados.0.smithi033.stdout:615: done (3 left) 2024-09-21T09:53:06.489 INFO:tasks.rados.rados.0.smithi033.stdout:616: setattr oid 4 current snap is 52 2024-09-21T09:53:06.489 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 52 2024-09-21T09:53:06.489 INFO:tasks.rados.rados.0.smithi033.stdout:617: seq_num 218 ranges {778240=458752,1236992=581632,1818624=221184} 2024-09-21T09:53:06.494 INFO:tasks.rados.rados.0.smithi033.stdout:617: writing smithi03385152-26 from 778240 to 1236992 tid 1 2024-09-21T09:53:06.498 INFO:tasks.rados.rados.0.smithi033.stdout:617: writing smithi03385152-26 from 1236992 to 1818624 tid 2 2024-09-21T09:53:06.500 INFO:tasks.rados.rados.0.smithi033.stdout:617: writing smithi03385152-26 from 1818624 to 2039808 tid 3 2024-09-21T09:53:06.501 INFO:tasks.rados.rados.0.smithi033.stdout:618: read oid 24 snap -1 2024-09-21T09:53:06.501 INFO:tasks.rados.rados.0.smithi033.stdout:618: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:53:06.501 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 52 2024-09-21T09:53:06.501 INFO:tasks.rados.rados.0.smithi033.stdout:619: seq_num 219 ranges {1818624=475136,2293760=466944,2760704=335872} 2024-09-21T09:53:06.513 INFO:tasks.rados.rados.0.smithi033.stdout:619: writing smithi03385152-3 from 1818624 to 2293760 tid 1 2024-09-21T09:53:06.516 INFO:tasks.rados.rados.0.smithi033.stdout:619: writing smithi03385152-3 from 2293760 to 2760704 tid 2 2024-09-21T09:53:06.518 INFO:tasks.rados.rados.0.smithi033.stdout:619: writing smithi03385152-3 from 2760704 to 3096576 tid 3 2024-09-21T09:53:06.519 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 52 2024-09-21T09:53:06.519 INFO:tasks.rados.rados.0.smithi033.stdout:620: seq_num 220 ranges {1671168=491520,2162688=753664,2916352=49152} 2024-09-21T09:53:06.530 INFO:tasks.rados.rados.0.smithi033.stdout:620: writing smithi03385152-25 from 1671168 to 2162688 tid 1 2024-09-21T09:53:06.535 INFO:tasks.rados.rados.0.smithi033.stdout:620: writing smithi03385152-25 from 2162688 to 2916352 tid 2 2024-09-21T09:53:06.535 INFO:tasks.rados.rados.0.smithi033.stdout:620: writing smithi03385152-25 from 2916352 to 2965504 tid 3 2024-09-21T09:53:06.535 INFO:tasks.rados.rados.0.smithi033.stdout:621: read oid 8 snap 30 2024-09-21T09:53:06.536 INFO:tasks.rados.rados.0.smithi033.stdout:621: expect deleted 2024-09-21T09:53:06.536 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 52 2024-09-21T09:53:06.536 INFO:tasks.rados.rados.0.smithi033.stdout:622: seq_num 221 ranges {1196032=507904,1703936=688128,2392064=114688} 2024-09-21T09:53:06.546 INFO:tasks.rados.rados.0.smithi033.stdout:622: writing smithi03385152-29 from 1196032 to 1703936 tid 1 2024-09-21T09:53:06.550 INFO:tasks.rados.rados.0.smithi033.stdout:622: writing smithi03385152-29 from 1703936 to 2392064 tid 2 2024-09-21T09:53:06.552 INFO:tasks.rados.rados.0.smithi033.stdout:622: writing smithi03385152-29 from 2392064 to 2506752 tid 3 2024-09-21T09:53:06.552 INFO:tasks.rados.rados.0.smithi033.stdout:623: rmattr oid 35 current snap is 52 2024-09-21T09:53:06.552 INFO:tasks.rados.rados.0.smithi033.stdout:623: done (10 left) 2024-09-21T09:53:06.552 INFO:tasks.rados.rados.0.smithi033.stdout:624: rmattr oid 31 current snap is 52 2024-09-21T09:53:06.552 INFO:tasks.rados.rados.0.smithi033.stdout:624: done (10 left) 2024-09-21T09:53:06.552 INFO:tasks.rados.rados.0.smithi033.stdout:625: rmattr oid 12 current snap is 52 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:613: done (10 left) 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:614: done (9 left) 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:625: done (8 left) 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 52 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:612: finishing rollback tid 0 to smithi03385152-47 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:612: finishing rollback tid 1 to smithi03385152-47 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:612: finishing rollback tid 2 to smithi03385152-47 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 38 (ObjNum 46 snap 0 seq_num 46) dirty exists 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 76 (ObjNum 33 snap 0 seq_num 90919904) dirty exists 2024-09-21T09:53:06.553 INFO:tasks.rados.rados.0.smithi033.stdout:626: seq_num 222 ranges {0=524288,524288=565248,1089536=237568} 2024-09-21T09:53:06.555 INFO:tasks.rados.rados.0.smithi033.stdout:626: writing smithi03385152-7 from 0 to 524288 tid 1 2024-09-21T09:53:06.559 INFO:tasks.rados.rados.0.smithi033.stdout:626: writing smithi03385152-7 from 524288 to 1089536 tid 2 2024-09-21T09:53:06.561 INFO:tasks.rados.rados.0.smithi033.stdout:626: writing smithi03385152-7 from 1089536 to 1327104 tid 3 2024-09-21T09:53:06.561 INFO:tasks.rados.rados.0.smithi033.stdout:612: done (8 left) 2024-09-21T09:53:06.561 INFO:tasks.rados.rados.0.smithi033.stdout:616: done (7 left) 2024-09-21T09:53:06.561 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 52 2024-09-21T09:53:06.561 INFO:tasks.rados.rados.0.smithi033.stdout:617: finishing write tid 1 to smithi03385152-26 2024-09-21T09:53:06.561 INFO:tasks.rados.rados.0.smithi033.stdout:627: seq_num 223 ranges {1933312=540672,2473984=450560,2924544=352256} 2024-09-21T09:53:06.574 INFO:tasks.rados.rados.0.smithi033.stdout:627: writing smithi03385152-22 from 1933312 to 2473984 tid 1 2024-09-21T09:53:06.577 INFO:tasks.rados.rados.0.smithi033.stdout:627: writing smithi03385152-22 from 2473984 to 2924544 tid 2 2024-09-21T09:53:06.579 INFO:tasks.rados.rados.0.smithi033.stdout:627: writing smithi03385152-22 from 2924544 to 3276800 tid 3 2024-09-21T09:53:06.579 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 52 2024-09-21T09:53:06.579 INFO:tasks.rados.rados.0.smithi033.stdout:628: seq_num 224 ranges {827392=557056,1384448=737280,2121728=65536} 2024-09-21T09:53:06.586 INFO:tasks.rados.rados.0.smithi033.stdout:628: writing smithi03385152-50 from 827392 to 1384448 tid 1 2024-09-21T09:53:06.591 INFO:tasks.rados.rados.0.smithi033.stdout:628: writing smithi03385152-50 from 1384448 to 2121728 tid 2 2024-09-21T09:53:06.591 INFO:tasks.rados.rados.0.smithi033.stdout:628: writing smithi03385152-50 from 2121728 to 2187264 tid 3 2024-09-21T09:53:06.592 INFO:tasks.rados.rados.0.smithi033.stdout:629: read oid 19 snap -1 2024-09-21T09:53:06.592 INFO:tasks.rados.rados.0.smithi033.stdout:629: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:53:06.592 INFO:tasks.rados.rados.0.smithi033.stdout:630: setattr oid 49 current snap is 52 2024-09-21T09:53:06.592 INFO:tasks.rados.rados.0.smithi033.stdout:617: finishing write tid 2 to smithi03385152-26 2024-09-21T09:53:06.592 INFO:tasks.rados.rados.0.smithi033.stdout:617: finishing write tid 3 to smithi03385152-26 2024-09-21T09:53:06.592 INFO:tasks.rados.rados.0.smithi033.stdout:617: finishing write tid 4 to smithi03385152-26 2024-09-21T09:53:06.592 INFO:tasks.rados.rados.0.smithi033.stdout:618: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:53:06.634 INFO:tasks.rados.rados.0.smithi033.stdout:617: finishing write tid 5 to smithi03385152-26 2024-09-21T09:53:06.634 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 242 (ObjNum 218 snap 52 seq_num 218) dirty exists 2024-09-21T09:53:06.634 INFO:tasks.rados.rados.0.smithi033.stdout:617: left oid 26 (ObjNum 218 snap 52 seq_num 218) 2024-09-21T09:53:06.634 INFO:tasks.rados.rados.0.smithi033.stdout:619: finishing write tid 1 to smithi03385152-3 2024-09-21T09:53:06.634 INFO:tasks.rados.rados.0.smithi033.stdout:619: finishing write tid 2 to smithi03385152-3 2024-09-21T09:53:06.634 INFO:tasks.rados.rados.0.smithi033.stdout:619: finishing write tid 3 to smithi03385152-3 2024-09-21T09:53:06.634 INFO:tasks.rados.rados.0.smithi033.stdout:619: finishing write tid 4 to smithi03385152-3 2024-09-21T09:53:06.634 INFO:tasks.rados.rados.0.smithi033.stdout:619: finishing write tid 5 to smithi03385152-3 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 336 (ObjNum 219 snap 52 seq_num 219) dirty exists 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:619: left oid 3 (ObjNum 219 snap 52 seq_num 219) 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:620: finishing write tid 1 to smithi03385152-25 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:620: finishing write tid 2 to smithi03385152-25 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:620: finishing write tid 3 to smithi03385152-25 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:620: finishing write tid 4 to smithi03385152-25 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:620: finishing write tid 5 to smithi03385152-25 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 71 (ObjNum 220 snap 52 seq_num 220) dirty exists 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:620: left oid 25 (ObjNum 220 snap 52 seq_num 220) 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:622: finishing write tid 1 to smithi03385152-29 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:622: finishing write tid 2 to smithi03385152-29 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:622: finishing write tid 3 to smithi03385152-29 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:622: finishing write tid 4 to smithi03385152-29 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:622: finishing write tid 5 to smithi03385152-29 2024-09-21T09:53:06.635 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 373 (ObjNum 221 snap 52 seq_num 221) dirty exists 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:622: left oid 29 (ObjNum 221 snap 52 seq_num 221) 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:617: done (10 left) 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:618: done (9 left) 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:619: done (8 left) 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:620: done (7 left) 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:621: done (6 left) 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:622: done (5 left) 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:631: snap_create 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:626: finishing write tid 1 to smithi03385152-7 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:626: finishing write tid 2 to smithi03385152-7 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:626: finishing write tid 3 to smithi03385152-7 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:626: finishing write tid 4 to smithi03385152-7 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:626: finishing write tid 5 to smithi03385152-7 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 378 (ObjNum 222 snap 52 seq_num 222) dirty exists 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:626: left oid 7 (ObjNum 222 snap 52 seq_num 222) 2024-09-21T09:53:06.636 INFO:tasks.rados.rados.0.smithi033.stdout:627: finishing write tid 1 to smithi03385152-22 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:627: finishing write tid 2 to smithi03385152-22 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:627: finishing write tid 3 to smithi03385152-22 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:627: finishing write tid 4 to smithi03385152-22 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:627: finishing write tid 5 to smithi03385152-22 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 341 (ObjNum 223 snap 52 seq_num 223) dirty exists 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:627: left oid 22 (ObjNum 223 snap 52 seq_num 223) 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:628: finishing write tid 1 to smithi03385152-50 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:629: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:628: finishing write tid 2 to smithi03385152-50 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:628: finishing write tid 3 to smithi03385152-50 2024-09-21T09:53:06.637 INFO:tasks.rados.rados.0.smithi033.stdout:628: finishing write tid 4 to smithi03385152-50 2024-09-21T09:53:06.638 INFO:tasks.rados.rados.0.smithi033.stdout:628: finishing write tid 5 to smithi03385152-50 2024-09-21T09:53:06.638 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 76 (ObjNum 224 snap 52 seq_num 224) dirty exists 2024-09-21T09:53:06.638 INFO:tasks.rados.rados.0.smithi033.stdout:628: left oid 50 (ObjNum 224 snap 52 seq_num 224) 2024-09-21T09:53:06.638 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 244 (ObjNum 98 snap 17 seq_num 98) dirty exists 2024-09-21T09:53:06.676 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:06 smithi033 systemd[1]: Stopping Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:06.677 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:06 smithi033 ceph-mon[27969]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:53:06.677 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:06 smithi033 ceph-mon[27969]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:53:06.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:06 smithi033 ceph-mon[39045]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:53:06.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:06 smithi033 ceph-mon[39045]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:53:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:06 smithi105 ceph-mon[31286]: osdmap e170: 8 total, 8 up, 8 in 2024-09-21T09:53:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:06 smithi105 ceph-mon[31286]: osdmap e171: 8 total, 8 up, 8 in 2024-09-21T09:53:06.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[80230]: ts=2024-09-21T09:53:06.675Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:53:06.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:06 smithi033 podman[87397]: 2024-09-21 09:53:06.705921447 +0000 UTC m=+0.141469944 container died f5c871f256757f8d960aea793f66ff9256062d2ac60c69b2e150ec8eacc9e910 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:07.393 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:07 smithi033 podman[87397]: 2024-09-21 09:53:07.139906372 +0000 UTC m=+0.575454854 container cleanup f5c871f256757f8d960aea793f66ff9256062d2ac60c69b2e150ec8eacc9e910 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:07.393 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:07 smithi033 bash[87397]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:626: done (5 left) 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:627: done (4 left) 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:628: done (3 left) 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:629: done (2 left) 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:630: done (1 left) 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:631: done (0 left) 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:632: rollback oid 32 current snap is 53 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 32 to 42 2024-09-21T09:53:07.540 INFO:tasks.rados.rados.0.smithi033.stdout:633: snap_remove snap 50 2024-09-21T09:53:07.721 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:07 smithi033 ceph-mon[39045]: pgmap v33: 121 pgs: 121 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-21T09:53:07.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:07 smithi033 ceph-mon[27969]: pgmap v33: 121 pgs: 121 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-21T09:53:07.721 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:07 smithi033 podman[87416]: 2024-09-21 09:53:07.391787951 +0000 UTC m=+0.680473233 container remove f5c871f256757f8d960aea793f66ff9256062d2ac60c69b2e150ec8eacc9e910 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:07.722 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:07 smithi033 podman[87416]: 2024-09-21 09:53:07.467524792 +0000 UTC m=+0.756210069 volume remove 4eb5781d6440dee94e91d84e022a729658909cf6cc02c0dc0c83b3c1b8a3e1ff 2024-09-21T09:53:07.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:07 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T09:53:07.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:07 smithi033 systemd[1]: Stopped Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:07.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:07 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 1.127s CPU time. 2024-09-21T09:53:07.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:07 smithi033 systemd[1]: Starting Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:07 smithi105 ceph-mon[31286]: pgmap v33: 121 pgs: 121 active+clean; 144 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 1.2 MiB/s wr, 26 op/s 2024-09-21T09:53:08.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 podman[87549]: 2024-09-21 09:53:08.055365694 +0000 UTC m=+0.016741191 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-21T09:53:08.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 podman[87549]: 2024-09-21 09:53:08.185267024 +0000 UTC m=+0.146642515 volume create 9dbf7a68c2ae6b2754434ca98eb7844214a3b83f98fb78c472ffa547d01874ed 2024-09-21T09:53:08.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 podman[87549]: 2024-09-21 09:53:08.342925158 +0000 UTC m=+0.304300644 container create 9ab68da7db7a9e5850b99ad6b2710c271f6b9ce06c15a22cacca4dae83a50e20 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:633: done (1 left) 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:634: rollback oid 46 current snap is 53 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 46 to 45 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:632: finishing rollback tid 0 to smithi03385152-32 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:632: finishing rollback tid 1 to smithi03385152-32 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:632: finishing rollback tid 2 to smithi03385152-32 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 57 (ObjNum 179 snap 40 seq_num 179) dirty exists 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:632: done (1 left) 2024-09-21T09:53:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:635: snap_remove snap 29 2024-09-21T09:53:08.920 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-mon[27969]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:53:08.920 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-mon[39045]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 podman[87549]: 2024-09-21 09:53:08.663722094 +0000 UTC m=+0.625097593 container init 9ab68da7db7a9e5850b99ad6b2710c271f6b9ce06c15a22cacca4dae83a50e20 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 podman[87549]: 2024-09-21 09:53:08.669000069 +0000 UTC m=+0.630375559 container start 9ab68da7db7a9e5850b99ad6b2710c271f6b9ce06c15a22cacca4dae83a50e20 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:08.679Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:08.679Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:08.680Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.33 port=9094 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:08.682Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:08.705Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:08.705Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:08.707Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:08.707Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 bash[87549]: 9ab68da7db7a9e5850b99ad6b2710c271f6b9ce06c15a22cacca4dae83a50e20 2024-09-21T09:53:08.921 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:08 smithi033 systemd[1]: Started Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:08 smithi105 ceph-mon[31286]: osdmap e172: 8 total, 8 up, 8 in 2024-09-21T09:53:09.222 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:53:08] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:53:09.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:09 smithi033 systemd[1]: Stopping Ceph node-exporter.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:09.556 INFO:tasks.rados.rados.0.smithi033.stdout:635: done (1 left) 2024-09-21T09:53:09.556 INFO:tasks.rados.rados.0.smithi033.stdout:636: read oid 21 snap -1 2024-09-21T09:53:09.556 INFO:tasks.rados.rados.0.smithi033.stdout:636: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:53:09.556 INFO:tasks.rados.rados.0.smithi033.stdout:637: rollback oid 32 current snap is 53 2024-09-21T09:53:09.557 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 32 to 51 2024-09-21T09:53:09.557 INFO:tasks.rados.rados.0.smithi033.stdout:638: copy_from oid 27 from oid 8 current snap is 53 2024-09-21T09:53:09.557 INFO:tasks.rados.rados.0.smithi033.stdout:639: read oid 38 snap -1 2024-09-21T09:53:09.557 INFO:tasks.rados.rados.0.smithi033.stdout:639: expect (ObjNum 188 snap 44 seq_num 188) 2024-09-21T09:53:09.557 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 53 2024-09-21T09:53:09.557 INFO:tasks.rados.rados.0.smithi033.stdout:640: seq_num 225 ranges {65536=573440,638976=622592,1261568=180224} 2024-09-21T09:53:09.558 INFO:tasks.rados.rados.0.smithi033.stdout:640: writing smithi03385152-30 from 65536 to 638976 tid 1 2024-09-21T09:53:09.563 INFO:tasks.rados.rados.0.smithi033.stdout:640: writing smithi03385152-30 from 638976 to 1261568 tid 2 2024-09-21T09:53:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:640: writing smithi03385152-30 from 1261568 to 1441792 tid 3 2024-09-21T09:53:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:634: finishing rollback tid 0 to smithi03385152-46 2024-09-21T09:53:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:634: finishing rollback tid 1 to smithi03385152-46 2024-09-21T09:53:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:634: finishing rollback tid 2 to smithi03385152-46 2024-09-21T09:53:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 345 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:53:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:638: finishing copy_from to smithi03385152-27 2024-09-21T09:53:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:638: got expected ENOENT (src dne) 2024-09-21T09:53:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:638: finishing copy_from racing read to smithi03385152-27 2024-09-21T09:53:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:634: done (5 left) 2024-09-21T09:53:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:638: done (4 left) 2024-09-21T09:53:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 53 2024-09-21T09:53:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:641: seq_num 226 ranges {2195456=589824,2785280=507904,3293184=294912} 2024-09-21T09:53:09.583 INFO:tasks.rados.rados.0.smithi033.stdout:641: writing smithi03385152-6 from 2195456 to 2785280 tid 1 2024-09-21T09:53:09.587 INFO:tasks.rados.rados.0.smithi033.stdout:641: writing smithi03385152-6 from 2785280 to 3293184 tid 2 2024-09-21T09:53:09.589 INFO:tasks.rados.rados.0.smithi033.stdout:641: writing smithi03385152-6 from 3293184 to 3588096 tid 3 2024-09-21T09:53:09.589 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 53 2024-09-21T09:53:09.590 INFO:tasks.rados.rados.0.smithi033.stdout:637: finishing rollback tid 0 to smithi03385152-32 2024-09-21T09:53:09.590 INFO:tasks.rados.rados.0.smithi033.stdout:637: finishing rollback tid 1 to smithi03385152-32 2024-09-21T09:53:09.590 INFO:tasks.rados.rados.0.smithi033.stdout:637: finishing rollback tid 2 to smithi03385152-32 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 62 (ObjNum 192 snap 45 seq_num 192) dirty exists 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:640: finishing write tid 1 to smithi03385152-30 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:640: finishing write tid 2 to smithi03385152-30 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:640: finishing write tid 3 to smithi03385152-30 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:640: finishing write tid 4 to smithi03385152-30 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:640: finishing write tid 5 to smithi03385152-30 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 391 (ObjNum 225 snap 53 seq_num 225) dirty exists 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:640: left oid 30 (ObjNum 225 snap 53 seq_num 225) 2024-09-21T09:53:09.591 INFO:tasks.rados.rados.0.smithi033.stdout:636: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:53:09.724 INFO:tasks.rados.rados.0.smithi033.stdout:641: finishing write tid 1 to smithi03385152-6 2024-09-21T09:53:09.724 INFO:tasks.rados.rados.0.smithi033.stdout:639: expect (ObjNum 188 snap 44 seq_num 188) 2024-09-21T09:53:09.953 INFO:tasks.rados.rados.0.smithi033.stdout:641: finishing write tid 2 to smithi03385152-6 2024-09-21T09:53:09.953 INFO:tasks.rados.rados.0.smithi033.stdout:641: finishing write tid 3 to smithi03385152-6 2024-09-21T09:53:09.953 INFO:tasks.rados.rados.0.smithi033.stdout:641: finishing write tid 4 to smithi03385152-6 2024-09-21T09:53:09.953 INFO:tasks.rados.rados.0.smithi033.stdout:641: finishing write tid 5 to smithi03385152-6 2024-09-21T09:53:09.953 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 197 (ObjNum 226 snap 53 seq_num 226) dirty exists 2024-09-21T09:53:09.954 INFO:tasks.rados.rados.0.smithi033.stdout:641: left oid 6 (ObjNum 226 snap 53 seq_num 226) 2024-09-21T09:53:09.954 INFO:tasks.rados.rados.0.smithi033.stdout:642: seq_num 227 ranges {1859584=606208,2465792=794624,3260416=8192} 2024-09-21T09:53:09.966 INFO:tasks.rados.rados.0.smithi033.stdout:642: writing smithi03385152-33 from 1859584 to 2465792 tid 1 2024-09-21T09:53:09.971 INFO:tasks.rados.rados.0.smithi033.stdout:642: writing smithi03385152-33 from 2465792 to 3260416 tid 2 2024-09-21T09:53:09.971 INFO:tasks.rados.rados.0.smithi033.stdout:642: writing smithi03385152-33 from 3260416 to 3268608 tid 3 2024-09-21T09:53:09.971 INFO:tasks.rados.rados.0.smithi033.stdout:636: done (5 left) 2024-09-21T09:53:09.971 INFO:tasks.rados.rados.0.smithi033.stdout:637: done (4 left) 2024-09-21T09:53:09.972 INFO:tasks.rados.rados.0.smithi033.stdout:639: done (3 left) 2024-09-21T09:53:09.972 INFO:tasks.rados.rados.0.smithi033.stdout:640: done (2 left) 2024-09-21T09:53:09.972 INFO:tasks.rados.rados.0.smithi033.stdout:641: done (1 left) 2024-09-21T09:53:09.972 INFO:tasks.rados.rados.0.smithi033.stdout:643: copy_from oid 11 from oid 32 current snap is 53 2024-09-21T09:53:09.972 INFO:tasks.rados.rados.0.smithi033.stdout:642: finishing write tid 1 to smithi03385152-33 2024-09-21T09:53:09.972 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 53 2024-09-21T09:53:09.972 INFO:tasks.rados.rados.0.smithi033.stdout:644: seq_num 228 ranges {0=622592,622592=671744,1294336=131072} 2024-09-21T09:53:09.974 INFO:tasks.rados.rados.0.smithi033.stdout:644: writing smithi03385152-2 from 0 to 622592 tid 1 2024-09-21T09:53:09.979 INFO:tasks.rados.rados.0.smithi033.stdout:644: writing smithi03385152-2 from 622592 to 1294336 tid 2 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:644: writing smithi03385152-2 from 1294336 to 1425408 tid 3 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:645: delete oid 47 current snap is 53 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:642: finishing write tid 2 to smithi03385152-33 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:642: finishing write tid 3 to smithi03385152-33 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:642: finishing write tid 4 to smithi03385152-33 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:644: finishing write tid 1 to smithi03385152-2 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:642: finishing write tid 5 to smithi03385152-33 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 202 (ObjNum 227 snap 53 seq_num 227) dirty exists 2024-09-21T09:53:09.980 INFO:tasks.rados.rados.0.smithi033.stdout:642: left oid 33 (ObjNum 227 snap 53 seq_num 227) 2024-09-21T09:53:09.982 INFO:tasks.rados.rados.0.smithi033.stdout:644: finishing write tid 2 to smithi03385152-2 2024-09-21T09:53:09.983 INFO:tasks.rados.rados.0.smithi033.stdout:644: finishing write tid 3 to smithi03385152-2 2024-09-21T09:53:09.983 INFO:tasks.rados.rados.0.smithi033.stdout:644: finishing write tid 4 to smithi03385152-2 2024-09-21T09:53:09.984 INFO:tasks.rados.rados.0.smithi033.stdout:642: done (3 left) 2024-09-21T09:53:09.984 INFO:tasks.rados.rados.0.smithi033.stdout:645: done (2 left) 2024-09-21T09:53:09.984 INFO:tasks.rados.rados.0.smithi033.stdout:646: snap_remove snap 52 2024-09-21T09:53:09.996 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:09 smithi033 podman[87713]: 2024-09-21 09:53:09.713967676 +0000 UTC m=+0.124427544 container died ae258c372f477343af831e4b4e311019a262336dcf96bb41a2daa9537fefbc12 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[39045]: pgmap v36: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.6 MiB/s wr, 54 op/s 2024-09-21T09:53:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[39045]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:53:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[39045]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:53:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[39045]: Deploying daemon node-exporter.a on smithi033 2024-09-21T09:53:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[27969]: pgmap v36: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.6 MiB/s wr, 54 op/s 2024-09-21T09:53:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[27969]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:53:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[27969]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:53:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:09 smithi033 ceph-mon[27969]: Deploying daemon node-exporter.a on smithi033 2024-09-21T09:53:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:09 smithi105 ceph-mon[31286]: pgmap v36: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.6 MiB/s wr, 54 op/s 2024-09-21T09:53:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:09 smithi105 ceph-mon[31286]: osdmap e173: 8 total, 8 up, 8 in 2024-09-21T09:53:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:09 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:09 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:09 smithi105 ceph-mon[31286]: Reconfiguring node-exporter.a (dependencies changed)... 2024-09-21T09:53:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:09 smithi105 ceph-mon[31286]: Deploying daemon node-exporter.a on smithi033 2024-09-21T09:53:10.345 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 podman[87713]: 2024-09-21 09:53:10.10131642 +0000 UTC m=+0.511776281 container cleanup ae258c372f477343af831e4b4e311019a262336dcf96bb41a2daa9537fefbc12 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:10.345 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 bash[87713]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T09:53:10.564 INFO:tasks.rados.rados.0.smithi033.stdout:646: done (2 left) 2024-09-21T09:53:10.565 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 53 2024-09-21T09:53:10.565 INFO:tasks.rados.rados.0.smithi033.stdout:647: seq_num 230 ranges {0=655360,655360=491520,1146880=311296} 2024-09-21T09:53:10.569 INFO:tasks.rados.rados.0.smithi033.stdout:647: writing smithi03385152-15 from 0 to 655360 tid 1 2024-09-21T09:53:10.573 INFO:tasks.rados.rados.0.smithi033.stdout:647: writing smithi03385152-15 from 655360 to 1146880 tid 2 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:647: writing smithi03385152-15 from 1146880 to 1458176 tid 3 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:648: read oid 12 snap -1 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:648: expect (ObjNum 177 snap 40 seq_num 177) 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:649: snap_create 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:644: finishing write tid 5 to smithi03385152-2 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 92 (ObjNum 228 snap 53 seq_num 228) dirty exists 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:644: left oid 2 (ObjNum 228 snap 53 seq_num 228) 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:643: finishing copy_from racing read to smithi03385152-11 2024-09-21T09:53:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:643: finishing copy_from to smithi03385152-11 2024-09-21T09:53:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 62 (ObjNum 192 snap 45 seq_num 192) dirty exists 2024-09-21T09:53:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:647: finishing write tid 1 to smithi03385152-15 2024-09-21T09:53:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:647: finishing write tid 2 to smithi03385152-15 2024-09-21T09:53:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:647: finishing write tid 3 to smithi03385152-15 2024-09-21T09:53:10.584 INFO:tasks.rados.rados.0.smithi033.stdout:647: finishing write tid 4 to smithi03385152-15 2024-09-21T09:53:10.585 INFO:tasks.rados.rados.0.smithi033.stdout:647: finishing write tid 5 to smithi03385152-15 2024-09-21T09:53:10.585 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 166 (ObjNum 230 snap 53 seq_num 230) dirty exists 2024-09-21T09:53:10.585 INFO:tasks.rados.rados.0.smithi033.stdout:647: left oid 15 (ObjNum 230 snap 53 seq_num 230) 2024-09-21T09:53:10.591 INFO:tasks.rados.rados.0.smithi033.stdout:648: expect (ObjNum 177 snap 40 seq_num 177) 2024-09-21T09:53:10.684 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:10 smithi033 ceph-mon[39045]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:53:10.685 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:10 smithi033 ceph-mon[27969]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:53:10.685 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 podman[87730]: 2024-09-21 09:53:10.344286918 +0000 UTC m=+0.633770084 container remove ae258c372f477343af831e4b4e311019a262336dcf96bb41a2daa9537fefbc12 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:10.685 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:53:10.685 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T09:53:10.685 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 systemd[1]: Stopped Ceph node-exporter.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:10.685 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 2.311s CPU time. 2024-09-21T09:53:10.996 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:10 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:10.682Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000265296s 2024-09-21T09:53:10.996 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 systemd[1]: Starting Ceph node-exporter.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:10.996 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:10 smithi033 bash[87859]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:53:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:10 smithi105 ceph-mon[31286]: osdmap e174: 8 total, 8 up, 8 in 2024-09-21T09:53:11.483 INFO:tasks.rados.rados.0.smithi033.stdout:643: done (4 left) 2024-09-21T09:53:11.483 INFO:tasks.rados.rados.0.smithi033.stdout:644: done (3 left) 2024-09-21T09:53:11.483 INFO:tasks.rados.rados.0.smithi033.stdout:647: done (2 left) 2024-09-21T09:53:11.483 INFO:tasks.rados.rados.0.smithi033.stdout:648: done (1 left) 2024-09-21T09:53:11.483 INFO:tasks.rados.rados.0.smithi033.stdout:649: done (0 left) 2024-09-21T09:53:11.483 INFO:tasks.rados.rados.0.smithi033.stdout:650: setattr oid 33 current snap is 54 2024-09-21T09:53:11.484 INFO:tasks.rados.rados.0.smithi033.stdout:651: read oid 6 snap -1 2024-09-21T09:53:11.484 INFO:tasks.rados.rados.0.smithi033.stdout:651: expect (ObjNum 226 snap 53 seq_num 226) 2024-09-21T09:53:11.484 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 54 2024-09-21T09:53:11.484 INFO:tasks.rados.rados.0.smithi033.stdout:652: seq_num 231 ranges {2523136=671744,3194880=778240,3973120=24576} 2024-09-21T09:53:11.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:11 smithi033 bash[87859]: Getting image source signatures 2024-09-21T09:53:11.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:11 smithi033 bash[87859]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:53:11.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:11 smithi033 bash[87859]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:53:11.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:11 smithi033 bash[87859]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:53:11.502 INFO:tasks.rados.rados.0.smithi033.stdout:652: writing smithi03385152-27 from 2523136 to 3194880 tid 1 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:652: writing smithi03385152-27 from 3194880 to 3973120 tid 2 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:652: writing smithi03385152-27 from 3973120 to 3997696 tid 3 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:653: read oid 46 snap -1 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:653: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:654: read oid 44 snap 53 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:654: expect (ObjNum 197 snap 46 seq_num 197) 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:655: setattr oid 48 current snap is 54 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 208 (ObjNum 227 snap 53 seq_num 227) dirty exists 2024-09-21T09:53:11.507 INFO:tasks.rados.rados.0.smithi033.stdout:651: expect (ObjNum 226 snap 53 seq_num 226) 2024-09-21T09:53:11.687 INFO:tasks.rados.rados.0.smithi033.stdout:652: finishing write tid 1 to smithi03385152-27 2024-09-21T09:53:11.688 INFO:tasks.rados.rados.0.smithi033.stdout:653: expect (ObjNum 1 snap 0 seq_num 1) 2024-09-21T09:53:11.688 INFO:tasks.rados.rados.0.smithi033.stdout:652: finishing write tid 2 to smithi03385152-27 2024-09-21T09:53:11.688 INFO:tasks.rados.rados.0.smithi033.stdout:652: finishing write tid 3 to smithi03385152-27 2024-09-21T09:53:11.688 INFO:tasks.rados.rados.0.smithi033.stdout:652: finishing write tid 4 to smithi03385152-27 2024-09-21T09:53:11.688 INFO:tasks.rados.rados.0.smithi033.stdout:652: finishing write tid 5 to smithi03385152-27 2024-09-21T09:53:11.689 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 213 (ObjNum 231 snap 54 seq_num 231) dirty exists 2024-09-21T09:53:11.689 INFO:tasks.rados.rados.0.smithi033.stdout:652: left oid 27 (ObjNum 231 snap 54 seq_num 231) 2024-09-21T09:53:11.689 INFO:tasks.rados.rados.0.smithi033.stdout:654: expect (ObjNum 197 snap 46 seq_num 197) 2024-09-21T09:53:11.714 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 369 (ObjNum 206 snap 50 seq_num 206) dirty exists 2024-09-21T09:53:11.714 INFO:tasks.rados.rados.0.smithi033.stdout:650: done (5 left) 2024-09-21T09:53:11.714 INFO:tasks.rados.rados.0.smithi033.stdout:651: done (4 left) 2024-09-21T09:53:11.714 INFO:tasks.rados.rados.0.smithi033.stdout:652: done (3 left) 2024-09-21T09:53:11.714 INFO:tasks.rados.rados.0.smithi033.stdout:653: done (2 left) 2024-09-21T09:53:11.715 INFO:tasks.rados.rados.0.smithi033.stdout:654: done (1 left) 2024-09-21T09:53:11.715 INFO:tasks.rados.rados.0.smithi033.stdout:655: done (0 left) 2024-09-21T09:53:11.715 INFO:tasks.rados.rados.0.smithi033.stdout:656: snap_remove snap 53 2024-09-21T09:53:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:11 smithi105 ceph-mon[31286]: pgmap v39: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.6 MiB/s wr, 54 op/s 2024-09-21T09:53:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:11 smithi105 ceph-mon[31286]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:53:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:11 smithi105 ceph-mon[31286]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:53:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:11 smithi033 ceph-mon[27969]: pgmap v39: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.6 MiB/s wr, 54 op/s 2024-09-21T09:53:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:11 smithi033 ceph-mon[27969]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:53:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:11 smithi033 ceph-mon[27969]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:53:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:11 smithi033 ceph-mon[39045]: pgmap v39: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 4.6 MiB/s wr, 54 op/s 2024-09-21T09:53:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:11 smithi033 ceph-mon[39045]: osdmap e175: 8 total, 8 up, 8 in 2024-09-21T09:53:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:11 smithi033 ceph-mon[39045]: osdmap e176: 8 total, 8 up, 8 in 2024-09-21T09:53:12.566 INFO:tasks.rados.rados.0.smithi033.stdout:656: done (0 left) 2024-09-21T09:53:12.566 INFO:tasks.rados.rados.0.smithi033.stdout:657: delete oid 43 current snap is 54 2024-09-21T09:53:12.569 INFO:tasks.rados.rados.0.smithi033.stdout:657: done (0 left) 2024-09-21T09:53:12.569 INFO:tasks.rados.rados.0.smithi033.stdout:658: snap_create 2024-09-21T09:53:12.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:12 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:53:12] "GET /metrics HTTP/1.1" 200 36209 "" "Prometheus/2.43.0" 2024-09-21T09:53:12.898 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:12 smithi033 bash[87859]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:53:12.898 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:12 smithi033 bash[87859]: Writing manifest to image destination 2024-09-21T09:53:13.184 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 podman[87859]: 2024-09-21 09:53:12.983057833 +0000 UTC m=+2.179425829 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:53:13.211 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 podman[87859]: 2024-09-21 09:53:13.072796954 +0000 UTC m=+2.269164938 container create cce524477a32bf23f87c2f1e136097a4865637dde62b79b35bc39e45ea32f0df (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:13.463 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 podman[87859]: 2024-09-21 09:53:13.267099013 +0000 UTC m=+2.463467007 container init cce524477a32bf23f87c2f1e136097a4865637dde62b79b35bc39e45ea32f0df (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 podman[87859]: 2024-09-21 09:53:13.272632042 +0000 UTC m=+2.469000026 container start cce524477a32bf23f87c2f1e136097a4865637dde62b79b35bc39e45ea32f0df (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.275Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.275Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.276Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.276Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.276Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.276Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:53:13.464 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:53:13.465 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:53:13.465 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:53:13.465 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:53:13.465 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:53:13.465 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:53:13.465 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:53:13.465 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:53:13.466 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:53:13.467 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a[87935]: ts=2024-09-21T09:53:13.277Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 bash[87859]: cce524477a32bf23f87c2f1e136097a4865637dde62b79b35bc39e45ea32f0df 2024-09-21T09:53:13.468 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 09:53:13 smithi033 systemd[1]: Started Ceph node-exporter.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:13.567 INFO:tasks.rados.rados.0.smithi033.stdout:658: done (0 left) 2024-09-21T09:53:13.567 INFO:tasks.rados.rados.0.smithi033.stdout:659: setattr oid 43 current snap is 55 2024-09-21T09:53:13.568 INFO:tasks.rados.rados.0.smithi033.stdout:660: snap_remove snap 41 2024-09-21T09:53:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-mon[39045]: pgmap v42: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-mon[39045]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:53:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:13.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:13.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-mon[27969]: pgmap v42: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:13.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-mon[27969]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:53:13.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:13.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:13 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:13 smithi105 ceph-mon[31286]: pgmap v42: 121 pgs: 121 active+clean; 166 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:13 smithi105 ceph-mon[31286]: osdmap e177: 8 total, 8 up, 8 in 2024-09-21T09:53:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:13 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:13 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:14.568 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 150 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:53:14.568 INFO:tasks.rados.rados.0.smithi033.stdout:659: done (1 left) 2024-09-21T09:53:14.568 INFO:tasks.rados.rados.0.smithi033.stdout:660: done (0 left) 2024-09-21T09:53:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:661: read oid 32 snap -1 2024-09-21T09:53:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:661: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:53:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:662: snap_remove snap 51 2024-09-21T09:53:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:14 smithi033 ceph-mon[27969]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:14 smithi033 ceph-mon[27969]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:53:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:14 smithi033 ceph-mon[27969]: Deploying daemon prometheus.a on smithi105 2024-09-21T09:53:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:14 smithi033 ceph-mon[27969]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:53:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:14 smithi033 ceph-mon[39045]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:14 smithi033 ceph-mon[39045]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:53:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:14 smithi033 ceph-mon[39045]: Deploying daemon prometheus.a on smithi105 2024-09-21T09:53:14.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:14 smithi033 ceph-mon[39045]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:53:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:14 smithi105 ceph-mon[31286]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:53:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:14 smithi105 ceph-mon[31286]: osdmap e178: 8 total, 8 up, 8 in 2024-09-21T09:53:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:14 smithi105 ceph-mon[31286]: Deploying daemon prometheus.a on smithi105 2024-09-21T09:53:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:14 smithi105 ceph-mon[31286]: osdmap e179: 8 total, 8 up, 8 in 2024-09-21T09:53:15.571 INFO:tasks.rados.rados.0.smithi033.stdout:662: done (1 left) 2024-09-21T09:53:15.571 INFO:tasks.rados.rados.0.smithi033.stdout:663: copy_from oid 9 from oid 40 current snap is 55 2024-09-21T09:53:15.571 INFO:tasks.rados.rados.0.smithi033.stdout:661: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:53:15.732 INFO:tasks.rados.rados.0.smithi033.stdout:663: finishing copy_from racing read to smithi03385152-9 2024-09-21T09:53:15.732 INFO:tasks.rados.rados.0.smithi033.stdout:663: finishing copy_from to smithi03385152-9 2024-09-21T09:53:15.733 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 140 (ObjNum 114 snap 25 seq_num 114) dirty exists 2024-09-21T09:53:15.733 INFO:tasks.rados.rados.0.smithi033.stdout:661: done (1 left) 2024-09-21T09:53:15.733 INFO:tasks.rados.rados.0.smithi033.stdout:663: done (0 left) 2024-09-21T09:53:15.733 INFO:tasks.rados.rados.0.smithi033.stdout:664: copy_from oid 25 from oid 2 current snap is 55 2024-09-21T09:53:15.733 INFO:tasks.rados.rados.0.smithi033.stdout:665: snap_remove snap 54 2024-09-21T09:53:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:15 smithi033 ceph-mon[39045]: pgmap v45: 121 pgs: 121 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-21T09:53:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:15 smithi033 ceph-mon[39045]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:53:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:15 smithi033 ceph-mon[27969]: pgmap v45: 121 pgs: 121 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-21T09:53:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:15 smithi033 ceph-mon[27969]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:53:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:15 smithi105 ceph-mon[31286]: pgmap v45: 121 pgs: 121 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-21T09:53:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:15 smithi105 ceph-mon[31286]: osdmap e180: 8 total, 8 up, 8 in 2024-09-21T09:53:16.486 INFO:tasks.rados.rados.0.smithi033.stdout:665: done (1 left) 2024-09-21T09:53:16.487 INFO:tasks.rados.rados.0.smithi033.stdout:666: delete oid 23 current snap is 55 2024-09-21T09:53:16.487 INFO:tasks.rados.rados.0.smithi033.stdout:664: finishing copy_from racing read to smithi03385152-25 2024-09-21T09:53:16.487 INFO:tasks.rados.rados.0.smithi033.stdout:664: finishing copy_from to smithi03385152-25 2024-09-21T09:53:16.487 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 92 (ObjNum 228 snap 53 seq_num 228) dirty exists 2024-09-21T09:53:16.491 INFO:tasks.rados.rados.0.smithi033.stdout:664: done (1 left) 2024-09-21T09:53:16.491 INFO:tasks.rados.rados.0.smithi033.stdout:666: done (0 left) 2024-09-21T09:53:16.491 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 55 2024-09-21T09:53:16.492 INFO:tasks.rados.rados.0.smithi033.stdout:667: seq_num 234 ranges {1302528=729088,2031616=434176,2465792=368640} 2024-09-21T09:53:16.503 INFO:tasks.rados.rados.0.smithi033.stdout:667: writing smithi03385152-24 from 1302528 to 2031616 tid 1 2024-09-21T09:53:16.505 INFO:tasks.rados.rados.0.smithi033.stdout:667: writing smithi03385152-24 from 2031616 to 2465792 tid 2 2024-09-21T09:53:16.508 INFO:tasks.rados.rados.0.smithi033.stdout:667: writing smithi03385152-24 from 2465792 to 2834432 tid 3 2024-09-21T09:53:16.508 INFO:tasks.rados.rados.0.smithi033.stdout:668: copy_from oid 49 from oid 22 current snap is 55 2024-09-21T09:53:16.508 INFO:tasks.rados.rados.0.smithi033.stdout:667: finishing write tid 1 to smithi03385152-24 2024-09-21T09:53:16.508 INFO:tasks.rados.rados.0.smithi033.stdout:667: finishing write tid 2 to smithi03385152-24 2024-09-21T09:53:16.508 INFO:tasks.rados.rados.0.smithi033.stdout:669: delete oid 35 current snap is 55 2024-09-21T09:53:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:667: finishing write tid 3 to smithi03385152-24 2024-09-21T09:53:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:667: finishing write tid 4 to smithi03385152-24 2024-09-21T09:53:16.511 INFO:tasks.rados.rados.0.smithi033.stdout:667: finishing write tid 5 to smithi03385152-24 2024-09-21T09:53:16.511 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 94 (ObjNum 234 snap 55 seq_num 234) dirty exists 2024-09-21T09:53:16.511 INFO:tasks.rados.rados.0.smithi033.stdout:667: left oid 24 (ObjNum 234 snap 55 seq_num 234) 2024-09-21T09:53:16.516 INFO:tasks.rados.rados.0.smithi033.stdout:667: done (2 left) 2024-09-21T09:53:16.516 INFO:tasks.rados.rados.0.smithi033.stdout:669: done (1 left) 2024-09-21T09:53:16.516 INFO:tasks.rados.rados.0.smithi033.stdout:670: read oid 8 snap -1 2024-09-21T09:53:16.516 INFO:tasks.rados.rados.0.smithi033.stdout:670: expect deleted 2024-09-21T09:53:16.516 INFO:tasks.rados.rados.0.smithi033.stdout:671: snap_remove snap 40 2024-09-21T09:53:17.496 INFO:tasks.rados.rados.0.smithi033.stdout:671: done (2 left) 2024-09-21T09:53:17.496 INFO:tasks.rados.rados.0.smithi033.stdout:672: snap_remove snap 30 2024-09-21T09:53:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:17 smithi033 ceph-mon[39045]: pgmap v48: 121 pgs: 121 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-21T09:53:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:17 smithi033 ceph-mon[39045]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:53:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:17 smithi033 ceph-mon[27969]: pgmap v48: 121 pgs: 121 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-21T09:53:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:17 smithi033 ceph-mon[27969]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:53:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:17 smithi105 ceph-mon[31286]: pgmap v48: 121 pgs: 121 active+clean; 169 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-21T09:53:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:17 smithi105 ceph-mon[31286]: osdmap e181: 8 total, 8 up, 8 in 2024-09-21T09:53:18.504 INFO:tasks.rados.rados.0.smithi033.stdout:668: finishing copy_from racing read to smithi03385152-49 2024-09-21T09:53:18.504 INFO:tasks.rados.rados.0.smithi033.stdout:668: finishing copy_from to smithi03385152-49 2024-09-21T09:53:18.505 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 341 (ObjNum 223 snap 52 seq_num 223) dirty exists 2024-09-21T09:53:18.505 INFO:tasks.rados.rados.0.smithi033.stdout:668: done (2 left) 2024-09-21T09:53:18.505 INFO:tasks.rados.rados.0.smithi033.stdout:670: done (1 left) 2024-09-21T09:53:18.505 INFO:tasks.rados.rados.0.smithi033.stdout:672: done (0 left) 2024-09-21T09:53:18.505 INFO:tasks.rados.rados.0.smithi033.stdout:673: setattr oid 48 current snap is 55 2024-09-21T09:53:18.507 INFO:tasks.rados.rados.0.smithi033.stdout:674: copy_from oid 14 from oid 10 current snap is 55 2024-09-21T09:53:18.508 INFO:tasks.rados.rados.0.smithi033.stdout:675: rollback oid 18 current snap is 55 2024-09-21T09:53:18.508 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 45 2024-09-21T09:53:18.508 INFO:tasks.rados.rados.0.smithi033.stdout:676: read oid 35 snap -1 2024-09-21T09:53:18.508 INFO:tasks.rados.rados.0.smithi033.stdout:676: expect deleted 2024-09-21T09:53:18.509 INFO:tasks.rados.rados.0.smithi033.stdout:677: snap_remove snap 42 2024-09-21T09:53:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:18 smithi105 ceph-mon[31286]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:53:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:18 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:18 smithi033 ceph-mon[39045]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:53:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:18 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:18.923 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 09:53:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T09:53:18.685Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003266699s 2024-09-21T09:53:18.924 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:18 smithi033 ceph-mon[27969]: osdmap e182: 8 total, 8 up, 8 in 2024-09-21T09:53:18.924 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:18 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:19.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:18 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[28222]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:53:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-21T09:53:19.503 INFO:tasks.rados.rados.0.smithi033.stdout:675: finishing rollback tid 0 to smithi03385152-18 2024-09-21T09:53:19.503 INFO:tasks.rados.rados.0.smithi033.stdout:674: finishing copy_from to smithi03385152-14 2024-09-21T09:53:19.503 INFO:tasks.rados.rados.0.smithi033.stdout:674: got expected ENOENT (src dne) 2024-09-21T09:53:19.503 INFO:tasks.rados.rados.0.smithi033.stdout:675: finishing rollback tid 1 to smithi03385152-18 2024-09-21T09:53:19.504 INFO:tasks.rados.rados.0.smithi033.stdout:675: finishing rollback tid 2 to smithi03385152-18 2024-09-21T09:53:19.504 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 116 (ObjNum 138 snap 32 seq_num 138) dirty exists 2024-09-21T09:53:19.504 INFO:tasks.rados.rados.0.smithi033.stdout:674: finishing copy_from racing read to smithi03385152-14 2024-09-21T09:53:19.504 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 412 (ObjNum 206 snap 50 seq_num 206) dirty exists 2024-09-21T09:53:19.504 INFO:tasks.rados.rados.0.smithi033.stdout:673: done (4 left) 2024-09-21T09:53:19.505 INFO:tasks.rados.rados.0.smithi033.stdout:674: done (3 left) 2024-09-21T09:53:19.505 INFO:tasks.rados.rados.0.smithi033.stdout:675: done (2 left) 2024-09-21T09:53:19.505 INFO:tasks.rados.rados.0.smithi033.stdout:676: done (1 left) 2024-09-21T09:53:19.505 INFO:tasks.rados.rados.0.smithi033.stdout:677: done (0 left) 2024-09-21T09:53:19.506 INFO:tasks.rados.rados.0.smithi033.stdout:678: copy_from oid 9 from oid 49 current snap is 55 2024-09-21T09:53:19.506 INFO:tasks.rados.rados.0.smithi033.stdout:679: setattr oid 33 current snap is 55 2024-09-21T09:53:19.508 INFO:tasks.rados.rados.0.smithi033.stdout:680: read oid 24 snap 45 2024-09-21T09:53:19.508 INFO:tasks.rados.rados.0.smithi033.stdout:680: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:53:19.508 INFO:tasks.rados.rados.0.smithi033.stdout:681: snap_create 2024-09-21T09:53:19.515 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 242 (ObjNum 227 snap 53 seq_num 227) dirty exists 2024-09-21T09:53:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:680: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:53:19.564 INFO:tasks.rados.rados.0.smithi033.stdout:678: finishing copy_from racing read to smithi03385152-9 2024-09-21T09:53:19.565 INFO:tasks.rados.rados.0.smithi033.stdout:678: finishing copy_from to smithi03385152-9 2024-09-21T09:53:19.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 341 (ObjNum 223 snap 52 seq_num 223) dirty exists 2024-09-21T09:53:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:19 smithi105 ceph-mon[31286]: pgmap v51: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:53:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:19 smithi105 ceph-mon[31286]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:53:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:19 smithi033 ceph-mon[27969]: pgmap v51: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:53:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:19 smithi033 ceph-mon[27969]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:53:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:19 smithi033 ceph-mon[39045]: pgmap v51: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:53:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:19 smithi033 ceph-mon[39045]: osdmap e183: 8 total, 8 up, 8 in 2024-09-21T09:53:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:678: done (3 left) 2024-09-21T09:53:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:679: done (2 left) 2024-09-21T09:53:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:680: done (1 left) 2024-09-21T09:53:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:681: done (0 left) 2024-09-21T09:53:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:682: setattr oid 5 current snap is 56 2024-09-21T09:53:20.519 INFO:tasks.rados.rados.0.smithi033.stdout:683: snap_create 2024-09-21T09:53:20.523 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 85 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-21T09:53:20.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:20 smithi105 ceph-mon[31286]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:53:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:20 smithi033 ceph-mon[39045]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:53:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:20 smithi033 ceph-mon[27969]: osdmap e184: 8 total, 8 up, 8 in 2024-09-21T09:53:21.486 INFO:tasks.rados.rados.0.smithi033.stdout:682: done (1 left) 2024-09-21T09:53:21.486 INFO:tasks.rados.rados.0.smithi033.stdout:683: done (0 left) 2024-09-21T09:53:21.486 INFO:tasks.rados.rados.0.smithi033.stdout:684: read oid 2 snap 55 2024-09-21T09:53:21.486 INFO:tasks.rados.rados.0.smithi033.stdout:684: expect (ObjNum 228 snap 53 seq_num 228) 2024-09-21T09:53:21.486 INFO:tasks.rados.rados.0.smithi033.stdout:685: read oid 33 snap -1 2024-09-21T09:53:21.486 INFO:tasks.rados.rados.0.smithi033.stdout:685: expect (ObjNum 227 snap 53 seq_num 227) 2024-09-21T09:53:21.487 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 57 2024-09-21T09:53:21.487 INFO:tasks.rados.rados.0.smithi033.stdout:686: seq_num 236 ranges {1425408=761856,2187264=647168,2834432=155648} 2024-09-21T09:53:21.498 INFO:tasks.rados.rados.0.smithi033.stdout:686: writing smithi03385152-25 from 1425408 to 2187264 tid 1 2024-09-21T09:53:21.502 INFO:tasks.rados.rados.0.smithi033.stdout:686: writing smithi03385152-25 from 2187264 to 2834432 tid 2 2024-09-21T09:53:21.504 INFO:tasks.rados.rados.0.smithi033.stdout:686: writing smithi03385152-25 from 2834432 to 2990080 tid 3 2024-09-21T09:53:21.504 INFO:tasks.rados.rados.0.smithi033.stdout:687: snap_create 2024-09-21T09:53:21.504 INFO:tasks.rados.rados.0.smithi033.stdout:684: expect (ObjNum 228 snap 53 seq_num 228) 2024-09-21T09:53:21.543 INFO:tasks.rados.rados.0.smithi033.stdout:686: finishing write tid 1 to smithi03385152-25 2024-09-21T09:53:21.543 INFO:tasks.rados.rados.0.smithi033.stdout:685: expect (ObjNum 227 snap 53 seq_num 227) 2024-09-21T09:53:21.676 INFO:tasks.rados.rados.0.smithi033.stdout:686: finishing write tid 2 to smithi03385152-25 2024-09-21T09:53:21.676 INFO:tasks.rados.rados.0.smithi033.stdout:686: finishing write tid 3 to smithi03385152-25 2024-09-21T09:53:21.676 INFO:tasks.rados.rados.0.smithi033.stdout:686: finishing write tid 4 to smithi03385152-25 2024-09-21T09:53:21.676 INFO:tasks.rados.rados.0.smithi033.stdout:686: finishing write tid 5 to smithi03385152-25 2024-09-21T09:53:21.676 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 129 (ObjNum 236 snap 57 seq_num 236) dirty exists 2024-09-21T09:53:21.676 INFO:tasks.rados.rados.0.smithi033.stdout:686: left oid 25 (ObjNum 236 snap 57 seq_num 236) 2024-09-21T09:53:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:21 smithi105 ceph-mon[31286]: pgmap v54: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:53:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:21 smithi105 ceph-mon[31286]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:53:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:21 smithi105 ceph-mon[31286]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:53:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:21 smithi033 ceph-mon[27969]: pgmap v54: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:53:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:21 smithi033 ceph-mon[27969]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:53:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:21 smithi033 ceph-mon[27969]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:53:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:21 smithi033 ceph-mon[39045]: pgmap v54: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.3 MiB/s rd, 2.1 MiB/s wr, 9 op/s 2024-09-21T09:53:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:21 smithi033 ceph-mon[39045]: osdmap e185: 8 total, 8 up, 8 in 2024-09-21T09:53:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:21 smithi033 ceph-mon[39045]: osdmap e186: 8 total, 8 up, 8 in 2024-09-21T09:53:22.489 INFO:tasks.rados.rados.0.smithi033.stdout:684: done (3 left) 2024-09-21T09:53:22.489 INFO:tasks.rados.rados.0.smithi033.stdout:685: done (2 left) 2024-09-21T09:53:22.489 INFO:tasks.rados.rados.0.smithi033.stdout:686: done (1 left) 2024-09-21T09:53:22.489 INFO:tasks.rados.rados.0.smithi033.stdout:687: done (0 left) 2024-09-21T09:53:22.489 INFO:tasks.rados.rados.0.smithi033.stdout:688: delete oid 9 current snap is 58 2024-09-21T09:53:22.491 INFO:tasks.rados.rados.0.smithi033.stdout:688: done (0 left) 2024-09-21T09:53:22.491 INFO:tasks.rados.rados.0.smithi033.stdout:689: snap_create 2024-09-21T09:53:22.695 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:53:22] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.43.0" 2024-09-21T09:53:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:689: done (0 left) 2024-09-21T09:53:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:690: rmattr oid 13 current snap is 59 2024-09-21T09:53:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:691: snap_create 2024-09-21T09:53:23.504 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 453 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:53:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:23 smithi033 ceph-mon[39045]: pgmap v57: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:23 smithi033 ceph-mon[39045]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:53:23.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:23 smithi033 ceph-mon[27969]: pgmap v57: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:23.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:23 smithi033 ceph-mon[27969]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:53:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:23 smithi105 ceph-mon[31286]: pgmap v57: 121 pgs: 121 active+clean; 168 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:23 smithi105 ceph-mon[31286]: osdmap e187: 8 total, 8 up, 8 in 2024-09-21T09:53:24.142 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:23 smithi105 systemd[1]: Stopping Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:24.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.140Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:53:24.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.140Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:53:24.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.140Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:53:24.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.140Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:53:24.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.140Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:53:24.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.140Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-21T09:53:24.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.140Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:53:24.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.140Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-21T09:53:24.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.141Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-21T09:53:24.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.141Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:53:24.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.141Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-21T09:53:24.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[64808]: ts=2024-09-21T09:53:24.141Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-21T09:53:24.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 podman[75601]: 2024-09-21 09:53:24.150982654 +0000 UTC m=+0.121289928 container died 9b3306609c6022938b2b57a80fa81c4ed174a6c41ec949ac2ec0c3f8fbcf0af6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:690: done (1 left) 2024-09-21T09:53:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:691: done (0 left) 2024-09-21T09:53:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:692: copy_from oid 43 from oid 39 current snap is 60 2024-09-21T09:53:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:693: delete oid 49 current snap is 60 2024-09-21T09:53:24.506 INFO:tasks.rados.rados.0.smithi033.stdout:693: done (1 left) 2024-09-21T09:53:24.507 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 60 2024-09-21T09:53:24.507 INFO:tasks.rados.rados.0.smithi033.stdout:694: seq_num 239 ranges {0=409600,409600=704512,1114112=499712} 2024-09-21T09:53:24.509 INFO:tasks.rados.rados.0.smithi033.stdout:694: writing smithi03385152-23 from 0 to 409600 tid 1 2024-09-21T09:53:24.513 INFO:tasks.rados.rados.0.smithi033.stdout:694: writing smithi03385152-23 from 409600 to 1114112 tid 2 2024-09-21T09:53:24.516 INFO:tasks.rados.rados.0.smithi033.stdout:694: writing smithi03385152-23 from 1114112 to 1613824 tid 3 2024-09-21T09:53:24.517 INFO:tasks.rados.rados.0.smithi033.stdout:695: rollback oid 3 current snap is 60 2024-09-21T09:53:24.517 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 3 to 55 2024-09-21T09:53:24.517 INFO:tasks.rados.rados.0.smithi033.stdout:696: snap_create 2024-09-21T09:53:24.517 INFO:tasks.rados.rados.0.smithi033.stdout:692: finishing copy_from racing read to smithi03385152-43 2024-09-21T09:53:24.517 INFO:tasks.rados.rados.0.smithi033.stdout:692: finishing copy_from to smithi03385152-43 2024-09-21T09:53:24.517 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 61 (ObjNum 199 snap 47 seq_num 199) dirty exists 2024-09-21T09:53:24.517 INFO:tasks.rados.rados.0.smithi033.stdout:694: finishing write tid 1 to smithi03385152-23 2024-09-21T09:53:24.519 INFO:tasks.rados.rados.0.smithi033.stdout:694: finishing write tid 2 to smithi03385152-23 2024-09-21T09:53:24.520 INFO:tasks.rados.rados.0.smithi033.stdout:694: finishing write tid 3 to smithi03385152-23 2024-09-21T09:53:24.520 INFO:tasks.rados.rados.0.smithi033.stdout:694: finishing write tid 4 to smithi03385152-23 2024-09-21T09:53:24.522 INFO:tasks.rados.rados.0.smithi033.stdout:695: finishing rollback tid 0 to smithi03385152-3 2024-09-21T09:53:24.522 INFO:tasks.rados.rados.0.smithi033.stdout:695: finishing rollback tid 1 to smithi03385152-3 2024-09-21T09:53:24.522 INFO:tasks.rados.rados.0.smithi033.stdout:695: finishing rollback tid 2 to smithi03385152-3 2024-09-21T09:53:24.522 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 440 (ObjNum 219 snap 52 seq_num 219) dirty exists 2024-09-21T09:53:24.523 INFO:tasks.rados.rados.0.smithi033.stdout:694: finishing write tid 5 to smithi03385152-23 2024-09-21T09:53:24.523 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 225 (ObjNum 239 snap 60 seq_num 239) dirty exists 2024-09-21T09:53:24.523 INFO:tasks.rados.rados.0.smithi033.stdout:694: left oid 23 (ObjNum 239 snap 60 seq_num 239) 2024-09-21T09:53:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:24 smithi105 ceph-mon[31286]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:53:24.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 podman[75601]: 2024-09-21 09:53:24.518650263 +0000 UTC m=+0.488957540 container cleanup 9b3306609c6022938b2b57a80fa81c4ed174a6c41ec949ac2ec0c3f8fbcf0af6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:24.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 bash[75601]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:53:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:24 smithi033 ceph-mon[39045]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:53:24.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:24 smithi033 ceph-mon[27969]: osdmap e188: 8 total, 8 up, 8 in 2024-09-21T09:53:25.145 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 podman[75618]: 2024-09-21 09:53:24.786891776 +0000 UTC m=+0.638574262 container remove 9b3306609c6022938b2b57a80fa81c4ed174a6c41ec949ac2ec0c3f8fbcf0af6 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:25.145 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:53:25.145 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 systemd[1]: Stopped Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:25.145 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:24 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.711s CPU time. 2024-09-21T09:53:25.498 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 systemd[1]: Starting Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:25.499 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 podman[75749]: 2024-09-21 09:53:25.254958089 +0000 UTC m=+0.016925445 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-21T09:53:25.499 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 podman[75749]: 2024-09-21 09:53:25.38027298 +0000 UTC m=+0.142257676 container create 12975b5508f873726f79426f72b84cdd18bb11bae88e5260c433d43acdec14fa (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:25.507 INFO:tasks.rados.rados.0.smithi033.stdout:692: done (3 left) 2024-09-21T09:53:25.507 INFO:tasks.rados.rados.0.smithi033.stdout:694: done (2 left) 2024-09-21T09:53:25.507 INFO:tasks.rados.rados.0.smithi033.stdout:695: done (1 left) 2024-09-21T09:53:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:696: done (0 left) 2024-09-21T09:53:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:697: read oid 42 snap -1 2024-09-21T09:53:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:697: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:53:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 61 2024-09-21T09:53:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:698: seq_num 240 ranges {2506752=425984,2932736=589824,3522560=483328,4005888=131072} 2024-09-21T09:53:25.523 INFO:tasks.rados.rados.0.smithi033.stdout:698: writing smithi03385152-29 from 2506752 to 2932736 tid 1 2024-09-21T09:53:25.527 INFO:tasks.rados.rados.0.smithi033.stdout:698: writing smithi03385152-29 from 2932736 to 3522560 tid 2 2024-09-21T09:53:25.530 INFO:tasks.rados.rados.0.smithi033.stdout:698: writing smithi03385152-29 from 3522560 to 4005888 tid 3 2024-09-21T09:53:25.531 INFO:tasks.rados.rados.0.smithi033.stdout:698: writing smithi03385152-29 from 4005888 to 4136960 tid 4 2024-09-21T09:53:25.531 INFO:tasks.rados.rados.0.smithi033.stdout:699: delete oid 3 current snap is 61 2024-09-21T09:53:25.531 INFO:tasks.rados.rados.0.smithi033.stdout:697: expect (ObjNum 54 snap 4 seq_num 54) 2024-09-21T09:53:25.575 INFO:tasks.rados.rados.0.smithi033.stdout:698: finishing write tid 1 to smithi03385152-29 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:698: finishing write tid 2 to smithi03385152-29 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:698: finishing write tid 3 to smithi03385152-29 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:698: finishing write tid 4 to smithi03385152-29 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:698: finishing write tid 5 to smithi03385152-29 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:698: finishing write tid 6 to smithi03385152-29 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 459 (ObjNum 240 snap 61 seq_num 240) dirty exists 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:698: left oid 29 (ObjNum 240 snap 61 seq_num 240) 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:697: done (2 left) 2024-09-21T09:53:25.576 INFO:tasks.rados.rados.0.smithi033.stdout:698: done (1 left) 2024-09-21T09:53:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:699: done (0 left) 2024-09-21T09:53:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:700: snap_create 2024-09-21T09:53:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-mon[31286]: pgmap v60: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:53:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-mon[31286]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:53:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:25 smithi033 ceph-mon[39045]: pgmap v60: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:53:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:25 smithi033 ceph-mon[39045]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:53:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:25 smithi033 ceph-mon[27969]: pgmap v60: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:53:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:25 smithi033 ceph-mon[27969]: osdmap e189: 8 total, 8 up, 8 in 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 podman[75749]: 2024-09-21 09:53:25.950636603 +0000 UTC m=+0.712603953 container init 12975b5508f873726f79426f72b84cdd18bb11bae88e5260c433d43acdec14fa (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 podman[75749]: 2024-09-21 09:53:25.957316458 +0000 UTC m=+0.719283793 container start 12975b5508f873726f79426f72b84cdd18bb11bae88e5260c433d43acdec14fa (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.978Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.978Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.978Z caller=main.go:623 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.978Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.978Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.980Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.981Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-21T09:53:26.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.982Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.983Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.984Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.984Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.775µs 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.984Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.984Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.997Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.997Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:25.997Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.883µs wal_replay_duration=12.68246ms wbl_replay_duration=116ns total_replay_duration=12.725447ms 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:26.000Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:26.000Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:26.000Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:26.016Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.575271ms db_storage=880ns remote_storage=1.192µs web_handler=639ns query_engine=825ns scrape=450.14µs scrape_sd=140.327µs notify=17.505µs notify_sd=14.632µs rules=15.568785ms tracing=6.166µs 2024-09-21T09:53:26.282 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:26.016Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-21T09:53:26.283 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:26.016Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:53:26.489 INFO:tasks.rados.rados.0.smithi033.stdout:700: done (0 left) 2024-09-21T09:53:26.489 INFO:tasks.rados.rados.0.smithi033.stdout:701: snap_create 2024-09-21T09:53:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:26 smithi105 ceph-mon[31286]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:53:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:26 smithi105 ceph-mon[31286]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:53:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:26 smithi033 ceph-mon[39045]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:53:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:26 smithi033 ceph-mon[39045]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:53:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:26 smithi033 ceph-mon[27969]: osdmap e190: 8 total, 8 up, 8 in 2024-09-21T09:53:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:26 smithi033 ceph-mon[27969]: osdmap e191: 8 total, 8 up, 8 in 2024-09-21T09:53:27.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:26 smithi105 bash[75749]: 12975b5508f873726f79426f72b84cdd18bb11bae88e5260c433d43acdec14fa 2024-09-21T09:53:27.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:26 smithi105 systemd[1]: Started Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:27.491 INFO:tasks.rados.rados.0.smithi033.stdout:701: done (0 left) 2024-09-21T09:53:27.492 INFO:tasks.rados.rados.0.smithi033.stdout:702: copy_from oid 48 from oid 1 current snap is 63 2024-09-21T09:53:27.492 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 63 2024-09-21T09:53:27.492 INFO:tasks.rados.rados.0.smithi033.stdout:703: seq_num 242 ranges {1548288=458752,2007040=753664,2760704=450560} 2024-09-21T09:53:27.503 INFO:tasks.rados.rados.0.smithi033.stdout:703: writing smithi03385152-41 from 1548288 to 2007040 tid 1 2024-09-21T09:53:27.508 INFO:tasks.rados.rados.0.smithi033.stdout:703: writing smithi03385152-41 from 2007040 to 2760704 tid 2 2024-09-21T09:53:27.511 INFO:tasks.rados.rados.0.smithi033.stdout:703: writing smithi03385152-41 from 2760704 to 3211264 tid 3 2024-09-21T09:53:27.511 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 63 2024-09-21T09:53:27.511 INFO:tasks.rados.rados.0.smithi033.stdout:704: seq_num 243 ranges {2990080=475136,3465216=638976,4104192=565248} 2024-09-21T09:53:27.530 INFO:tasks.rados.rados.0.smithi033.stdout:704: writing smithi03385152-25 from 2990080 to 3465216 tid 1 2024-09-21T09:53:27.533 INFO:tasks.rados.rados.0.smithi033.stdout:704: writing smithi03385152-25 from 3465216 to 4104192 tid 2 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:704: writing smithi03385152-25 from 4104192 to 4669440 tid 3 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:705: rmattr oid 32 current snap is 63 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:705: done (3 left) 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:706: read oid 36 snap -1 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:706: expect deleted 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 63 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:702: finishing copy_from racing read to smithi03385152-48 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:702: finishing copy_from to smithi03385152-48 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 147 (ObjNum 213 snap 51 seq_num 213) dirty exists 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:703: finishing write tid 1 to smithi03385152-41 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:703: finishing write tid 2 to smithi03385152-41 2024-09-21T09:53:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:703: finishing write tid 3 to smithi03385152-41 2024-09-21T09:53:27.539 INFO:tasks.rados.rados.0.smithi033.stdout:703: finishing write tid 4 to smithi03385152-41 2024-09-21T09:53:27.539 INFO:tasks.rados.rados.0.smithi033.stdout:703: finishing write tid 5 to smithi03385152-41 2024-09-21T09:53:27.539 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 346 (ObjNum 242 snap 63 seq_num 242) dirty exists 2024-09-21T09:53:27.539 INFO:tasks.rados.rados.0.smithi033.stdout:703: left oid 41 (ObjNum 242 snap 63 seq_num 242) 2024-09-21T09:53:27.539 INFO:tasks.rados.rados.0.smithi033.stdout:707: seq_num 244 ranges {0=491520,491520=573440,1064960=458752,1523712=172032} 2024-09-21T09:53:27.540 INFO:tasks.rados.rados.0.smithi033.stdout:707: writing smithi03385152-28 from 0 to 491520 tid 1 2024-09-21T09:53:27.543 INFO:tasks.rados.rados.0.smithi033.stdout:707: writing smithi03385152-28 from 491520 to 1064960 tid 2 2024-09-21T09:53:27.546 INFO:tasks.rados.rados.0.smithi033.stdout:707: writing smithi03385152-28 from 1064960 to 1523712 tid 3 2024-09-21T09:53:27.547 INFO:tasks.rados.rados.0.smithi033.stdout:707: writing smithi03385152-28 from 1523712 to 1695744 tid 4 2024-09-21T09:53:27.547 INFO:tasks.rados.rados.0.smithi033.stdout:704: finishing write tid 1 to smithi03385152-25 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:704: finishing write tid 2 to smithi03385152-25 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:704: finishing write tid 3 to smithi03385152-25 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:704: finishing write tid 4 to smithi03385152-25 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:702: done (4 left) 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:703: done (3 left) 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:706: done (2 left) 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:708: copy_from oid 38 from oid 45 current snap is 63 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:704: finishing write tid 5 to smithi03385152-25 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 134 (ObjNum 243 snap 63 seq_num 243) dirty exists 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:704: left oid 25 (ObjNum 243 snap 63 seq_num 243) 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:707: finishing write tid 1 to smithi03385152-28 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:707: finishing write tid 2 to smithi03385152-28 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:704: done (2 left) 2024-09-21T09:53:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:709: snap_create 2024-09-21T09:53:27.549 INFO:tasks.rados.rados.0.smithi033.stdout:707: finishing write tid 3 to smithi03385152-28 2024-09-21T09:53:27.549 INFO:tasks.rados.rados.0.smithi033.stdout:707: finishing write tid 4 to smithi03385152-28 2024-09-21T09:53:27.550 INFO:tasks.rados.rados.0.smithi033.stdout:707: finishing write tid 5 to smithi03385152-28 2024-09-21T09:53:27.551 INFO:tasks.rados.rados.0.smithi033.stdout:707: finishing write tid 6 to smithi03385152-28 2024-09-21T09:53:27.551 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 79 (ObjNum 244 snap 63 seq_num 244) dirty exists 2024-09-21T09:53:27.551 INFO:tasks.rados.rados.0.smithi033.stdout:707: left oid 28 (ObjNum 244 snap 63 seq_num 244) 2024-09-21T09:53:27.566 INFO:tasks.rados.rados.0.smithi033.stdout:708: finishing copy_from racing read to smithi03385152-38 2024-09-21T09:53:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:708: finishing copy_from to smithi03385152-38 2024-09-21T09:53:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 138 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:53:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:27 smithi105 ceph-mon[31286]: pgmap v63: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:53:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:27 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:27 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:27 smithi105 ceph-mon[31286]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:53:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:27 smithi033 ceph-mon[39045]: pgmap v63: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:53:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:27 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:27 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:27 smithi033 ceph-mon[39045]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:53:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:27 smithi033 ceph-mon[27969]: pgmap v63: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:53:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:27 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:27 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:27 smithi033 ceph-mon[27969]: osdmap e192: 8 total, 8 up, 8 in 2024-09-21T09:53:28.519 INFO:tasks.rados.rados.0.smithi033.stdout:707: done (2 left) 2024-09-21T09:53:28.519 INFO:tasks.rados.rados.0.smithi033.stdout:708: done (1 left) 2024-09-21T09:53:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:709: done (0 left) 2024-09-21T09:53:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:710: rollback oid 46 current snap is 64 2024-09-21T09:53:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 46 to 61 2024-09-21T09:53:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:711: copy_from oid 35 from oid 11 current snap is 64 2024-09-21T09:53:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:712: read oid 30 snap -1 2024-09-21T09:53:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:712: expect (ObjNum 225 snap 53 seq_num 225) 2024-09-21T09:53:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:713: copy_from oid 6 from oid 38 current snap is 64 2024-09-21T09:53:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:714: delete oid 33 current snap is 64 2024-09-21T09:53:28.527 INFO:tasks.rados.rados.0.smithi033.stdout:710: finishing rollback tid 0 to smithi03385152-46 2024-09-21T09:53:28.527 INFO:tasks.rados.rados.0.smithi033.stdout:714: done (4 left) 2024-09-21T09:53:28.527 INFO:tasks.rados.rados.0.smithi033.stdout:715: read oid 32 snap -1 2024-09-21T09:53:28.527 INFO:tasks.rados.rados.0.smithi033.stdout:715: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:53:28.527 INFO:tasks.rados.rados.0.smithi033.stdout:716: read oid 23 snap 55 2024-09-21T09:53:28.527 INFO:tasks.rados.rados.0.smithi033.stdout:716: expect deleted 2024-09-21T09:53:28.527 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 64 2024-09-21T09:53:28.528 INFO:tasks.rados.rados.0.smithi033.stdout:717: seq_num 246 ranges {3973120=524288,4497408=745472,5242880=425984,5668864=32768} 2024-09-21T09:53:28.553 INFO:tasks.rados.rados.0.smithi033.stdout:717: writing smithi03385152-31 from 3973120 to 4497408 tid 1 2024-09-21T09:53:28.557 INFO:tasks.rados.rados.0.smithi033.stdout:717: writing smithi03385152-31 from 4497408 to 5242880 tid 2 2024-09-21T09:53:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:717: writing smithi03385152-31 from 5242880 to 5668864 tid 3 2024-09-21T09:53:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:717: writing smithi03385152-31 from 5668864 to 5701632 tid 4 2024-09-21T09:53:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:718: rmattr oid 44 current snap is 64 2024-09-21T09:53:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:718: done (7 left) 2024-09-21T09:53:28.561 INFO:tasks.rados.rados.0.smithi033.stdout:719: snap_remove snap 62 2024-09-21T09:53:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:28 smithi105 ceph-mon[31286]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:53:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:28 smithi105 ceph-mon[31286]: Reconfiguring daemon grafana.a on smithi105 2024-09-21T09:53:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:28 smithi033 ceph-mon[27969]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:53:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:28 smithi033 ceph-mon[27969]: Reconfiguring daemon grafana.a on smithi105 2024-09-21T09:53:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:28 smithi033 ceph-mon[39045]: Reconfiguring grafana.a (dependencies changed)... 2024-09-21T09:53:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:28 smithi033 ceph-mon[39045]: Reconfiguring daemon grafana.a on smithi105 2024-09-21T09:53:29.527 INFO:tasks.rados.rados.0.smithi033.stdout:719: done (7 left) 2024-09-21T09:53:29.527 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 64 2024-09-21T09:53:29.527 INFO:tasks.rados.rados.0.smithi033.stdout:710: finishing rollback tid 1 to smithi03385152-46 2024-09-21T09:53:29.528 INFO:tasks.rados.rados.0.smithi033.stdout:710: finishing rollback tid 2 to smithi03385152-46 2024-09-21T09:53:29.528 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 448 (ObjNum 1 snap 0 seq_num 1) dirty exists 2024-09-21T09:53:29.528 INFO:tasks.rados.rados.0.smithi033.stdout:712: expect (ObjNum 225 snap 53 seq_num 225) 2024-09-21T09:53:29.569 INFO:tasks.rados.rados.0.smithi033.stdout:715: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:53:29.723 INFO:tasks.rados.rados.0.smithi033.stdout:711: finishing copy_from racing read to smithi03385152-35 2024-09-21T09:53:29.723 INFO:tasks.rados.rados.0.smithi033.stdout:713: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:711: finishing copy_from to smithi03385152-35 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 62 (ObjNum 192 snap 45 seq_num 192) dirty exists 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:717: finishing write tid 1 to smithi03385152-31 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:713: finishing copy_from to smithi03385152-6 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 138 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:717: finishing write tid 2 to smithi03385152-31 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:717: finishing write tid 3 to smithi03385152-31 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:717: finishing write tid 4 to smithi03385152-31 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:717: finishing write tid 5 to smithi03385152-31 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:717: finishing write tid 6 to smithi03385152-31 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 466 (ObjNum 246 snap 64 seq_num 246) dirty exists 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:717: left oid 31 (ObjNum 246 snap 64 seq_num 246) 2024-09-21T09:53:29.724 INFO:tasks.rados.rados.0.smithi033.stdout:720: seq_num 247 ranges {0=540672,540672=622592,1163264=581632} 2024-09-21T09:53:29.726 INFO:tasks.rados.rados.0.smithi033.stdout:720: writing smithi03385152-4 from 0 to 540672 tid 1 2024-09-21T09:53:29.729 INFO:tasks.rados.rados.0.smithi033.stdout:720: writing smithi03385152-4 from 540672 to 1163264 tid 2 2024-09-21T09:53:29.734 INFO:tasks.rados.rados.0.smithi033.stdout:720: writing smithi03385152-4 from 1163264 to 1744896 tid 3 2024-09-21T09:53:29.734 INFO:tasks.rados.rados.0.smithi033.stdout:710: done (7 left) 2024-09-21T09:53:29.734 INFO:tasks.rados.rados.0.smithi033.stdout:711: done (6 left) 2024-09-21T09:53:29.734 INFO:tasks.rados.rados.0.smithi033.stdout:712: done (5 left) 2024-09-21T09:53:29.734 INFO:tasks.rados.rados.0.smithi033.stdout:713: done (4 left) 2024-09-21T09:53:29.734 INFO:tasks.rados.rados.0.smithi033.stdout:715: done (3 left) 2024-09-21T09:53:29.734 INFO:tasks.rados.rados.0.smithi033.stdout:716: done (2 left) 2024-09-21T09:53:29.735 INFO:tasks.rados.rados.0.smithi033.stdout:717: done (1 left) 2024-09-21T09:53:29.735 INFO:tasks.rados.rados.0.smithi033.stdout:721: rollback oid 44 current snap is 64 2024-09-21T09:53:29.735 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 44 to 45 2024-09-21T09:53:29.735 INFO:tasks.rados.rados.0.smithi033.stdout:720: finishing write tid 1 to smithi03385152-4 2024-09-21T09:53:29.735 INFO:tasks.rados.rados.0.smithi033.stdout:722: snap_create 2024-09-21T09:53:29.735 INFO:tasks.rados.rados.0.smithi033.stdout:720: finishing write tid 2 to smithi03385152-4 2024-09-21T09:53:29.737 INFO:tasks.rados.rados.0.smithi033.stdout:720: finishing write tid 3 to smithi03385152-4 2024-09-21T09:53:29.737 INFO:tasks.rados.rados.0.smithi033.stdout:720: finishing write tid 4 to smithi03385152-4 2024-09-21T09:53:29.740 INFO:tasks.rados.rados.0.smithi033.stdout:721: finishing rollback tid 0 to smithi03385152-44 2024-09-21T09:53:29.742 INFO:tasks.rados.rados.0.smithi033.stdout:721: finishing rollback tid 1 to smithi03385152-44 2024-09-21T09:53:29.742 INFO:tasks.rados.rados.0.smithi033.stdout:720: finishing write tid 5 to smithi03385152-4 2024-09-21T09:53:29.743 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 93 (ObjNum 247 snap 64 seq_num 247) dirty exists 2024-09-21T09:53:29.743 INFO:tasks.rados.rados.0.smithi033.stdout:720: left oid 4 (ObjNum 247 snap 64 seq_num 247) 2024-09-21T09:53:29.743 INFO:tasks.rados.rados.0.smithi033.stdout:721: finishing rollback tid 2 to smithi03385152-44 2024-09-21T09:53:29.743 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 97 (ObjNum 139 snap 32 seq_num 139) dirty exists 2024-09-21T09:53:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:29 smithi105 ceph-mon[31286]: pgmap v66: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 976 KiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-21T09:53:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:29 smithi105 ceph-mon[31286]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:53:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:29 smithi033 ceph-mon[39045]: pgmap v66: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 976 KiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-21T09:53:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:29 smithi033 ceph-mon[39045]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:53:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:29 smithi033 ceph-mon[27969]: pgmap v66: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 976 KiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-21T09:53:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:29 smithi033 ceph-mon[27969]: osdmap e193: 8 total, 8 up, 8 in 2024-09-21T09:53:30.395 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:30 smithi105 systemd[1]: Stopping Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:30.536 INFO:tasks.rados.rados.0.smithi033.stdout:720: done (2 left) 2024-09-21T09:53:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:721: done (1 left) 2024-09-21T09:53:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:722: done (0 left) 2024-09-21T09:53:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:723: copy_from oid 48 from oid 7 current snap is 65 2024-09-21T09:53:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:724: read oid 2 snap -1 2024-09-21T09:53:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:724: expect (ObjNum 228 snap 53 seq_num 228) 2024-09-21T09:53:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 65 2024-09-21T09:53:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:725: seq_num 248 ranges {4161536=557056,4718592=507904,5226496=442368,5668864=253952} 2024-09-21T09:53:30.562 INFO:tasks.rados.rados.0.smithi033.stdout:725: writing smithi03385152-13 from 4161536 to 4718592 tid 1 2024-09-21T09:53:30.565 INFO:tasks.rados.rados.0.smithi033.stdout:725: writing smithi03385152-13 from 4718592 to 5226496 tid 2 2024-09-21T09:53:30.568 INFO:tasks.rados.rados.0.smithi033.stdout:725: writing smithi03385152-13 from 5226496 to 5668864 tid 3 2024-09-21T09:53:30.570 INFO:tasks.rados.rados.0.smithi033.stdout:725: writing smithi03385152-13 from 5668864 to 5922816 tid 4 2024-09-21T09:53:30.570 INFO:tasks.rados.rados.0.smithi033.stdout:726: read oid 47 snap -1 2024-09-21T09:53:30.570 INFO:tasks.rados.rados.0.smithi033.stdout:726: expect deleted 2024-09-21T09:53:30.570 INFO:tasks.rados.rados.0.smithi033.stdout:727: delete oid 26 current snap is 65 2024-09-21T09:53:30.570 INFO:tasks.rados.rados.0.smithi033.stdout:724: expect (ObjNum 228 snap 53 seq_num 228) 2024-09-21T09:53:30.608 INFO:tasks.rados.rados.0.smithi033.stdout:723: finishing copy_from racing read to smithi03385152-48 2024-09-21T09:53:30.609 INFO:tasks.rados.rados.0.smithi033.stdout:723: finishing copy_from to smithi03385152-48 2024-09-21T09:53:30.609 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 378 (ObjNum 222 snap 52 seq_num 222) dirty exists 2024-09-21T09:53:30.609 INFO:tasks.rados.rados.0.smithi033.stdout:725: finishing write tid 1 to smithi03385152-13 2024-09-21T09:53:30.609 INFO:tasks.rados.rados.0.smithi033.stdout:725: finishing write tid 2 to smithi03385152-13 2024-09-21T09:53:30.609 INFO:tasks.rados.rados.0.smithi033.stdout:723: done (4 left) 2024-09-21T09:53:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:724: done (3 left) 2024-09-21T09:53:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:727: done (2 left) 2024-09-21T09:53:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:728: read oid 45 snap -1 2024-09-21T09:53:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:728: expect (ObjNum 172 snap 37 seq_num 172) 2024-09-21T09:53:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 65 2024-09-21T09:53:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:729: seq_num 250 ranges {0=589824,589824=679936,1269760=409600,1679360=114688} 2024-09-21T09:53:30.611 INFO:tasks.rados.rados.0.smithi033.stdout:729: writing smithi03385152-16 from 0 to 589824 tid 1 2024-09-21T09:53:30.615 INFO:tasks.rados.rados.0.smithi033.stdout:729: writing smithi03385152-16 from 589824 to 1269760 tid 2 2024-09-21T09:53:30.617 INFO:tasks.rados.rados.0.smithi033.stdout:729: writing smithi03385152-16 from 1269760 to 1679360 tid 3 2024-09-21T09:53:30.619 INFO:tasks.rados.rados.0.smithi033.stdout:729: writing smithi03385152-16 from 1679360 to 1794048 tid 4 2024-09-21T09:53:30.619 INFO:tasks.rados.rados.0.smithi033.stdout:726: done (3 left) 2024-09-21T09:53:30.620 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 65 2024-09-21T09:53:30.620 INFO:tasks.rados.rados.0.smithi033.stdout:725: finishing write tid 3 to smithi03385152-13 2024-09-21T09:53:30.620 INFO:tasks.rados.rados.0.smithi033.stdout:725: finishing write tid 4 to smithi03385152-13 2024-09-21T09:53:30.620 INFO:tasks.rados.rados.0.smithi033.stdout:725: finishing write tid 5 to smithi03385152-13 2024-09-21T09:53:30.620 INFO:tasks.rados.rados.0.smithi033.stdout:725: finishing write tid 6 to smithi03385152-13 2024-09-21T09:53:30.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 474 (ObjNum 248 snap 65 seq_num 248) dirty exists 2024-09-21T09:53:30.621 INFO:tasks.rados.rados.0.smithi033.stdout:725: left oid 13 (ObjNum 248 snap 65 seq_num 248) 2024-09-21T09:53:30.621 INFO:tasks.rados.rados.0.smithi033.stdout:730: seq_num 251 ranges {2834432=606208,3440640=614400,4055040=589824} 2024-09-21T09:53:30.638 INFO:tasks.rados.rados.0.smithi033.stdout:730: writing smithi03385152-24 from 2834432 to 3440640 tid 1 2024-09-21T09:53:30.641 INFO:tasks.rados.rados.0.smithi033.stdout:730: writing smithi03385152-24 from 3440640 to 4055040 tid 2 2024-09-21T09:53:30.645 INFO:tasks.rados.rados.0.smithi033.stdout:730: writing smithi03385152-24 from 4055040 to 4644864 tid 3 2024-09-21T09:53:30.645 INFO:tasks.rados.rados.0.smithi033.stdout:725: done (3 left) 2024-09-21T09:53:30.646 INFO:tasks.rados.rados.0.smithi033.stdout:731: read oid 8 snap -1 2024-09-21T09:53:30.646 INFO:tasks.rados.rados.0.smithi033.stdout:731: expect deleted 2024-09-21T09:53:30.646 INFO:tasks.rados.rados.0.smithi033.stdout:732: copy_from oid 31 from oid 12 current snap is 65 2024-09-21T09:53:30.646 INFO:tasks.rados.rados.0.smithi033.stdout:728: expect (ObjNum 172 snap 37 seq_num 172) 2024-09-21T09:53:30.658 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:30 smithi105 ceph-mon[31286]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:53:30.658 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:30 smithi105 podman[76007]: 2024-09-21 09:53:30.419064434 +0000 UTC m=+0.123489699 container died 39f7f99588560a6b5c975934837c4f7b194df7ce1c4d30caf07965d21b442d58 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, io.openshift.tags=base rhel8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, version=8.5, vcs-type=git) 2024-09-21T09:53:30.848 INFO:tasks.rados.rados.0.smithi033.stdout:729: finishing write tid 1 to smithi03385152-16 2024-09-21T09:53:30.848 INFO:tasks.rados.rados.0.smithi033.stdout:729: finishing write tid 2 to smithi03385152-16 2024-09-21T09:53:30.848 INFO:tasks.rados.rados.0.smithi033.stdout:728: done (4 left) 2024-09-21T09:53:30.848 INFO:tasks.rados.rados.0.smithi033.stdout:733: snap_remove snap 60 2024-09-21T09:53:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:30 smithi033 ceph-mon[27969]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:53:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:30 smithi033 ceph-mon[39045]: osdmap e194: 8 total, 8 up, 8 in 2024-09-21T09:53:31.030 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:30 smithi105 podman[76007]: 2024-09-21 09:53:30.822700893 +0000 UTC m=+0.527126157 container cleanup 39f7f99588560a6b5c975934837c4f7b194df7ce1c4d30caf07965d21b442d58 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , version=8.5, architecture=x86_64, vcs-type=git, vendor=Red Hat, Inc., name=ubi8, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, release=239.1651231664, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8) 2024-09-21T09:53:31.030 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:30 smithi105 bash[76007]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a 2024-09-21T09:53:31.441 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 podman[76025]: 2024-09-21 09:53:31.061882542 +0000 UTC m=+0.642497334 container remove 39f7f99588560a6b5c975934837c4f7b194df7ce1c4d30caf07965d21b442d58 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, com.redhat.component=ubi8-container, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, architecture=x86_64, io.buildah.version=1.32.0, io.openshift.tags=base rhel8) 2024-09-21T09:53:31.441 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:53:31.442 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service: Failed with result 'exit-code'. 2024-09-21T09:53:31.442 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 systemd[1]: Stopped Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:31.442 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service: Consumed 5.289s CPU time. 2024-09-21T09:53:31.442 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 systemd[1]: Starting Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:31.542 INFO:tasks.rados.rados.0.smithi033.stdout:733: done (4 left) 2024-09-21T09:53:31.542 INFO:tasks.rados.rados.0.smithi033.stdout:734: delete oid 47 current snap is 65 2024-09-21T09:53:31.542 INFO:tasks.rados.rados.0.smithi033.stdout:729: finishing write tid 3 to smithi03385152-16 2024-09-21T09:53:31.542 INFO:tasks.rados.rados.0.smithi033.stdout:729: finishing write tid 4 to smithi03385152-16 2024-09-21T09:53:31.542 INFO:tasks.rados.rados.0.smithi033.stdout:729: finishing write tid 5 to smithi03385152-16 2024-09-21T09:53:31.542 INFO:tasks.rados.rados.0.smithi033.stdout:729: finishing write tid 6 to smithi03385152-16 2024-09-21T09:53:31.542 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 479 (ObjNum 250 snap 65 seq_num 250) dirty exists 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:729: left oid 16 (ObjNum 250 snap 65 seq_num 250) 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:730: finishing write tid 1 to smithi03385152-24 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:730: finishing write tid 2 to smithi03385152-24 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:730: finishing write tid 3 to smithi03385152-24 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:730: finishing write tid 4 to smithi03385152-24 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:730: finishing write tid 5 to smithi03385152-24 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 109 (ObjNum 251 snap 65 seq_num 251) dirty exists 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:730: left oid 24 (ObjNum 251 snap 65 seq_num 251) 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:732: finishing copy_from racing read to smithi03385152-31 2024-09-21T09:53:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:732: finishing copy_from to smithi03385152-31 2024-09-21T09:53:31.544 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 231 (ObjNum 177 snap 40 seq_num 177) dirty exists 2024-09-21T09:53:31.547 INFO:tasks.rados.rados.0.smithi033.stdout:729: done (4 left) 2024-09-21T09:53:31.547 INFO:tasks.rados.rados.0.smithi033.stdout:730: done (3 left) 2024-09-21T09:53:31.547 INFO:tasks.rados.rados.0.smithi033.stdout:731: done (2 left) 2024-09-21T09:53:31.547 INFO:tasks.rados.rados.0.smithi033.stdout:732: done (1 left) 2024-09-21T09:53:31.547 INFO:tasks.rados.rados.0.smithi033.stdout:734: done (0 left) 2024-09-21T09:53:31.547 INFO:tasks.rados.rados.0.smithi033.stdout:735: read oid 11 snap -1 2024-09-21T09:53:31.547 INFO:tasks.rados.rados.0.smithi033.stdout:735: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:53:31.547 INFO:tasks.rados.rados.0.smithi033.stdout:736: snap_remove snap 58 2024-09-21T09:53:31.691 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-mon[31286]: pgmap v69: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 976 KiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-21T09:53:31.692 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-mon[31286]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:53:31.692 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 podman[76133]: 2024-09-21 09:53:31.341788214 +0000 UTC m=+0.015316678 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-21T09:53:31.692 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 podman[76133]: 2024-09-21 09:53:31.459072943 +0000 UTC m=+0.132601398 container create 6f501c82dfe359e0f9c3e702828bad29f98f989ba91356456d5366e6adf44733 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.openshift.tags=base rhel8, io.openshift.expose-services=, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, distribution-scope=public, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container) 2024-09-21T09:53:31.692 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 podman[76133]: 2024-09-21 09:53:31.686660308 +0000 UTC m=+0.360188763 container init 6f501c82dfe359e0f9c3e702828bad29f98f989ba91356456d5366e6adf44733 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.openshift.tags=base rhel8, architecture=x86_64, io.openshift.expose-services=, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, version=8.5, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, maintainer=Paul Cuzner ) 2024-09-21T09:53:31.947 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:53:31] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.51.0" 2024-09-21T09:53:31.947 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 podman[76133]: 2024-09-21 09:53:31.692552315 +0000 UTC m=+0.366080764 container start 6f501c82dfe359e0f9c3e702828bad29f98f989ba91356456d5366e6adf44733 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, io.openshift.tags=base rhel8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, maintainer=Paul Cuzner , summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, vendor=Red Hat, Inc.) 2024-09-21T09:53:31.947 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844733531Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-21T09:53:31.947 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844932582Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844951772Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844959303Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844965517Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844971711Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844977442Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.84498345Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844989606Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.844995532Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.845001149Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T09:53:31.948 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.84500722Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.845013107Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=settings t=2024-09-21T09:53:31.845020309Z level=info msg="App mode production" 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=sqlstore t=2024-09-21T09:53:31.845072725Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=sqlstore t=2024-09-21T09:53:31.845089448Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=migrator t=2024-09-21T09:53:31.858594927Z level=info msg="Starting DB migrations" 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=migrator t=2024-09-21T09:53:31.863042369Z level=info msg="migrations completed" performed=0 skipped=481 duration=400.304µs 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=sqlstore t=2024-09-21T09:53:31.863362228Z level=info msg="Created default organization" 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=plugin.loader t=2024-09-21T09:53:31.919601532Z level=info msg="Plugin registered" pluginID=input 2024-09-21T09:53:31.949 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 bash[76133]: 6f501c82dfe359e0f9c3e702828bad29f98f989ba91356456d5366e6adf44733 2024-09-21T09:53:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:31 smithi033 ceph-mon[27969]: pgmap v69: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 976 KiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-21T09:53:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:31 smithi033 ceph-mon[27969]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:53:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:31 smithi033 ceph-mon[39045]: pgmap v69: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 976 KiB/s rd, 2.8 MiB/s wr, 11 op/s 2024-09-21T09:53:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:31 smithi033 ceph-mon[39045]: osdmap e195: 8 total, 8 up, 8 in 2024-09-21T09:53:32.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=plugin.loader t=2024-09-21T09:53:31.947685639Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-21T09:53:32.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=plugin.loader t=2024-09-21T09:53:31.947717805Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-21T09:53:32.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=secrets t=2024-09-21T09:53:31.947765265Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T09:53:32.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 systemd[1]: Started Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:32.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=query_data t=2024-09-21T09:53:31.949573949Z level=info msg="Query Service initialization" 2024-09-21T09:53:32.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=live.push_http t=2024-09-21T09:53:31.952516815Z level=info msg="Live Push Gateway initialization" 2024-09-21T09:53:32.549 INFO:tasks.rados.rados.0.smithi033.stdout:736: done (1 left) 2024-09-21T09:53:32.549 INFO:tasks.rados.rados.0.smithi033.stdout:737: read oid 12 snap -1 2024-09-21T09:53:32.549 INFO:tasks.rados.rados.0.smithi033.stdout:737: expect (ObjNum 177 snap 40 seq_num 177) 2024-09-21T09:53:32.549 INFO:tasks.rados.rados.0.smithi033.stdout:738: read oid 29 snap -1 2024-09-21T09:53:32.549 INFO:tasks.rados.rados.0.smithi033.stdout:738: expect (ObjNum 240 snap 61 seq_num 240) 2024-09-21T09:53:32.550 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 65 2024-09-21T09:53:32.550 INFO:tasks.rados.rados.0.smithi033.stdout:739: seq_num 253 ranges {0=647168,647168=786432,1433600=417792} 2024-09-21T09:53:32.554 INFO:tasks.rados.rados.0.smithi033.stdout:739: writing smithi03385152-17 from 0 to 647168 tid 1 2024-09-21T09:53:32.559 INFO:tasks.rados.rados.0.smithi033.stdout:739: writing smithi03385152-17 from 647168 to 1433600 tid 2 2024-09-21T09:53:32.561 INFO:tasks.rados.rados.0.smithi033.stdout:739: writing smithi03385152-17 from 1433600 to 1851392 tid 3 2024-09-21T09:53:32.561 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 65 2024-09-21T09:53:32.561 INFO:tasks.rados.rados.0.smithi033.stdout:740: seq_num 254 ranges {1327104=663552,1990656=663552,2654208=540672} 2024-09-21T09:53:32.573 INFO:tasks.rados.rados.0.smithi033.stdout:740: writing smithi03385152-7 from 1327104 to 1990656 tid 1 2024-09-21T09:53:32.577 INFO:tasks.rados.rados.0.smithi033.stdout:740: writing smithi03385152-7 from 1990656 to 2654208 tid 2 2024-09-21T09:53:32.580 INFO:tasks.rados.rados.0.smithi033.stdout:740: writing smithi03385152-7 from 2654208 to 3194880 tid 3 2024-09-21T09:53:32.580 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 65 2024-09-21T09:53:32.580 INFO:tasks.rados.rados.0.smithi033.stdout:741: seq_num 255 ranges {0=679936,679936=548864,1228800=598016,1826816=57344} 2024-09-21T09:53:32.584 INFO:tasks.rados.rados.0.smithi033.stdout:741: writing smithi03385152-8 from 0 to 679936 tid 1 2024-09-21T09:53:32.588 INFO:tasks.rados.rados.0.smithi033.stdout:741: writing smithi03385152-8 from 679936 to 1228800 tid 2 2024-09-21T09:53:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:741: writing smithi03385152-8 from 1228800 to 1826816 tid 3 2024-09-21T09:53:32.593 INFO:tasks.rados.rados.0.smithi033.stdout:741: writing smithi03385152-8 from 1826816 to 1884160 tid 4 2024-09-21T09:53:32.594 INFO:tasks.rados.rados.0.smithi033.stdout:742: rmattr oid 16 current snap is 65 2024-09-21T09:53:32.594 INFO:tasks.rados.rados.0.smithi033.stdout:742: done (6 left) 2024-09-21T09:53:32.594 INFO:tasks.rados.rados.0.smithi033.stdout:743: snap_remove snap 63 2024-09-21T09:53:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-mon[31286]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:53:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:32.781 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:32 smithi105 systemd[1]: Stopping Ceph node-exporter.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:32.781 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=infra.usagestats.collector t=2024-09-21T09:53:32.711336347Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T09:53:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:32 smithi033 ceph-mon[39045]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:53:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:32 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:32 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:32 smithi033 ceph-mon[27969]: osdmap e196: 8 total, 8 up, 8 in 2024-09-21T09:53:32.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:32 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:32.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:32 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:33.280 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:33 smithi105 podman[76297]: 2024-09-21 09:53:33.022098296 +0000 UTC m=+0.134146662 container died 446328798f333717972f40f1413ed20a3dc9f9a994b803faf822b21c19d67d64 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:53:33.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=provisioning.datasources t=2024-09-21T09:53:32.799078019Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=provisioning.datasources t=2024-09-21T09:53:32.799115867Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=provisioning.datasources t=2024-09-21T09:53:32.809303414Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=provisioning.alerting t=2024-09-21T09:53:32.896936139Z level=info msg="starting to provision alerting" 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=provisioning.alerting t=2024-09-21T09:53:32.896966789Z level=info msg="finished to provision alerting" 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=ngalert.state.manager t=2024-09-21T09:53:32.897133869Z level=info msg="Warming state cache for startup" 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=grafanaStorageLogger t=2024-09-21T09:53:32.897163784Z level=info msg="storage starting" 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=http.server t=2024-09-21T09:53:32.899621412Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=ngalert.state.manager t=2024-09-21T09:53:32.907839976Z level=info msg="State cache has been initialized" states=0 duration=10.702245ms 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=ticker t=2024-09-21T09:53:32.907933345Z level=info msg=starting first_tick=2024-09-21T09:53:40Z 2024-09-21T09:53:33.281 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 09:53:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[76148]: logger=ngalert.multiorg.alertmanager t=2024-09-21T09:53:32.907938614Z level=info msg="starting MultiOrg Alertmanager" 2024-09-21T09:53:33.551 INFO:tasks.rados.rados.0.smithi033.stdout:735: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:53:33.553 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:33 smithi105 podman[76297]: 2024-09-21 09:53:33.393048842 +0000 UTC m=+0.505097206 container cleanup 446328798f333717972f40f1413ed20a3dc9f9a994b803faf822b21c19d67d64 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:53:33.553 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:33 smithi105 bash[76297]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T09:53:33.707 INFO:tasks.rados.rados.0.smithi033.stdout:741: finishing write tid 1 to smithi03385152-8 2024-09-21T09:53:33.707 INFO:tasks.rados.rados.0.smithi033.stdout:737: expect (ObjNum 177 snap 40 seq_num 177) 2024-09-21T09:53:33.805 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:33 smithi105 podman[76316]: 2024-09-21 09:53:33.677911419 +0000 UTC m=+0.659768241 container remove 446328798f333717972f40f1413ed20a3dc9f9a994b803faf822b21c19d67d64 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:53:33.805 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:33 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:53:33.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:33 smithi105 ceph-mon[31286]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:53:33.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:33 smithi105 ceph-mon[31286]: Deploying daemon node-exporter.b on smithi105 2024-09-21T09:53:33.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:33 smithi105 ceph-mon[31286]: pgmap v72: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:33.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:33 smithi105 ceph-mon[31286]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:53:33.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:33 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:33.935 INFO:tasks.rados.rados.0.smithi033.stdout:739: finishing write tid 1 to smithi03385152-17 2024-09-21T09:53:33.935 INFO:tasks.rados.rados.0.smithi033.stdout:739: finishing write tid 2 to smithi03385152-17 2024-09-21T09:53:33.935 INFO:tasks.rados.rados.0.smithi033.stdout:739: finishing write tid 3 to smithi03385152-17 2024-09-21T09:53:33.935 INFO:tasks.rados.rados.0.smithi033.stdout:739: finishing write tid 4 to smithi03385152-17 2024-09-21T09:53:33.935 INFO:tasks.rados.rados.0.smithi033.stdout:739: finishing write tid 5 to smithi03385152-17 2024-09-21T09:53:33.935 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 347 (ObjNum 253 snap 65 seq_num 253) dirty exists 2024-09-21T09:53:33.935 INFO:tasks.rados.rados.0.smithi033.stdout:739: left oid 17 (ObjNum 253 snap 65 seq_num 253) 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:741: finishing write tid 2 to smithi03385152-8 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:735: done (6 left) 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:737: done (5 left) 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:739: done (4 left) 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:743: done (3 left) 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 65 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:741: finishing write tid 3 to smithi03385152-8 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:741: finishing write tid 4 to smithi03385152-8 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:741: finishing write tid 5 to smithi03385152-8 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:741: finishing write tid 6 to smithi03385152-8 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 193 (ObjNum 255 snap 65 seq_num 255) dirty exists 2024-09-21T09:53:33.936 INFO:tasks.rados.rados.0.smithi033.stdout:741: left oid 8 (ObjNum 255 snap 65 seq_num 255) 2024-09-21T09:53:33.937 INFO:tasks.rados.rados.0.smithi033.stdout:738: expect (ObjNum 240 snap 61 seq_num 240) 2024-09-21T09:53:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[39045]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:53:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[39045]: Deploying daemon node-exporter.b on smithi105 2024-09-21T09:53:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[39045]: pgmap v72: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[39045]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:53:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[27969]: Reconfiguring node-exporter.b (dependencies changed)... 2024-09-21T09:53:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[27969]: Deploying daemon node-exporter.b on smithi105 2024-09-21T09:53:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[27969]: pgmap v72: 121 pgs: 121 active+clean; 140 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[27969]: osdmap e197: 8 total, 8 up, 8 in 2024-09-21T09:53:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:33 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:34.104 INFO:tasks.rados.rados.0.smithi033.stdout:740: finishing write tid 1 to smithi03385152-7 2024-09-21T09:53:34.104 INFO:tasks.rados.rados.0.smithi033.stdout:740: finishing write tid 2 to smithi03385152-7 2024-09-21T09:53:34.104 INFO:tasks.rados.rados.0.smithi033.stdout:740: finishing write tid 3 to smithi03385152-7 2024-09-21T09:53:34.104 INFO:tasks.rados.rados.0.smithi033.stdout:740: finishing write tid 4 to smithi03385152-7 2024-09-21T09:53:34.104 INFO:tasks.rados.rados.0.smithi033.stdout:740: finishing write tid 5 to smithi03385152-7 2024-09-21T09:53:34.104 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 494 (ObjNum 254 snap 65 seq_num 254) dirty exists 2024-09-21T09:53:34.104 INFO:tasks.rados.rados.0.smithi033.stdout:740: left oid 7 (ObjNum 254 snap 65 seq_num 254) 2024-09-21T09:53:34.105 INFO:tasks.rados.rados.0.smithi033.stdout:744: seq_num 256 ranges {0=696320,696320=434176,1130496=770048} 2024-09-21T09:53:34.107 INFO:tasks.rados.rados.0.smithi033.stdout:744: writing smithi03385152-49 from 0 to 696320 tid 1 2024-09-21T09:53:34.109 INFO:tasks.rados.rados.0.smithi033.stdout:744: writing smithi03385152-49 from 696320 to 1130496 tid 2 2024-09-21T09:53:34.114 INFO:tasks.rados.rados.0.smithi033.stdout:744: writing smithi03385152-49 from 1130496 to 1900544 tid 3 2024-09-21T09:53:34.114 INFO:tasks.rados.rados.0.smithi033.stdout:738: done (3 left) 2024-09-21T09:53:34.114 INFO:tasks.rados.rados.0.smithi033.stdout:740: done (2 left) 2024-09-21T09:53:34.114 INFO:tasks.rados.rados.0.smithi033.stdout:741: done (1 left) 2024-09-21T09:53:34.114 INFO:tasks.rados.rados.0.smithi033.stdout:745: snap_create 2024-09-21T09:53:34.114 INFO:tasks.rados.rados.0.smithi033.stdout:744: finishing write tid 1 to smithi03385152-49 2024-09-21T09:53:34.114 INFO:tasks.rados.rados.0.smithi033.stdout:744: finishing write tid 2 to smithi03385152-49 2024-09-21T09:53:34.117 INFO:tasks.rados.rados.0.smithi033.stdout:744: finishing write tid 3 to smithi03385152-49 2024-09-21T09:53:34.117 INFO:tasks.rados.rados.0.smithi033.stdout:744: finishing write tid 4 to smithi03385152-49 2024-09-21T09:53:34.121 INFO:tasks.rados.rados.0.smithi033.stdout:744: finishing write tid 5 to smithi03385152-49 2024-09-21T09:53:34.121 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 351 (ObjNum 256 snap 65 seq_num 256) dirty exists 2024-09-21T09:53:34.121 INFO:tasks.rados.rados.0.smithi033.stdout:744: left oid 49 (ObjNum 256 snap 65 seq_num 256) 2024-09-21T09:53:34.148 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:33 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T09:53:34.148 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:33 smithi105 systemd[1]: Stopped Ceph node-exporter.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:34.148 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:33 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 2.193s CPU time. 2024-09-21T09:53:34.148 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:34 smithi105 systemd[1]: Starting Ceph node-exporter.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:34.530 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:34 smithi105 bash[76448]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-21T09:53:34.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:34.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:34.567 INFO:tasks.rados.rados.0.smithi033.stdout:744: done (1 left) 2024-09-21T09:53:34.567 INFO:tasks.rados.rados.0.smithi033.stdout:745: done (0 left) 2024-09-21T09:53:34.567 INFO:tasks.rados.rados.0.smithi033.stdout:746: delete oid 46 current snap is 66 2024-09-21T09:53:34.570 INFO:tasks.rados.rados.0.smithi033.stdout:746: done (0 left) 2024-09-21T09:53:34.570 INFO:tasks.rados.rados.0.smithi033.stdout:747: delete oid 44 current snap is 66 2024-09-21T09:53:34.572 INFO:tasks.rados.rados.0.smithi033.stdout:747: done (0 left) 2024-09-21T09:53:34.572 INFO:tasks.rados.rados.0.smithi033.stdout:748: read oid 46 snap -1 2024-09-21T09:53:34.572 INFO:tasks.rados.rados.0.smithi033.stdout:748: expect deleted 2024-09-21T09:53:34.572 INFO:tasks.rados.rados.0.smithi033.stdout:749: rollback oid 47 current snap is 66 2024-09-21T09:53:34.572 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 45 2024-09-21T09:53:34.573 INFO:tasks.rados.rados.0.smithi033.stdout:750: read oid 43 snap -1 2024-09-21T09:53:34.573 INFO:tasks.rados.rados.0.smithi033.stdout:750: expect (ObjNum 199 snap 47 seq_num 199) 2024-09-21T09:53:34.573 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 66 2024-09-21T09:53:34.573 INFO:tasks.rados.rados.0.smithi033.stdout:751: seq_num 259 ranges {3612672=745472,4358144=540672,4898816=581632,5480448=81920} 2024-09-21T09:53:34.596 INFO:tasks.rados.rados.0.smithi033.stdout:751: writing smithi03385152-31 from 3612672 to 4358144 tid 1 2024-09-21T09:53:34.599 INFO:tasks.rados.rados.0.smithi033.stdout:751: writing smithi03385152-31 from 4358144 to 4898816 tid 2 2024-09-21T09:53:34.603 INFO:tasks.rados.rados.0.smithi033.stdout:751: writing smithi03385152-31 from 4898816 to 5480448 tid 3 2024-09-21T09:53:34.604 INFO:tasks.rados.rados.0.smithi033.stdout:751: writing smithi03385152-31 from 5480448 to 5562368 tid 4 2024-09-21T09:53:34.604 INFO:tasks.rados.rados.0.smithi033.stdout:752: read oid 24 snap 57 2024-09-21T09:53:34.604 INFO:tasks.rados.rados.0.smithi033.stdout:752: expect (ObjNum 234 snap 55 seq_num 234) 2024-09-21T09:53:34.604 INFO:tasks.rados.rados.0.smithi033.stdout:753: rollback oid 10 current snap is 66 2024-09-21T09:53:34.604 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 56 2024-09-21T09:53:34.604 INFO:tasks.rados.rados.0.smithi033.stdout:754: read oid 12 snap -1 2024-09-21T09:53:34.604 INFO:tasks.rados.rados.0.smithi033.stdout:754: expect (ObjNum 177 snap 40 seq_num 177) 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:755: rollback oid 18 current snap is 66 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 55 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:749: finishing rollback tid 1 to smithi03385152-47 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:749: finishing rollback tid 2 to smithi03385152-47 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 61 (ObjNum 166 snap 37 seq_num 166) dirty exists 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:748: done (7 left) 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:749: done (6 left) 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:756: rollback oid 21 current snap is 66 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 65 2024-09-21T09:53:34.605 INFO:tasks.rados.rados.0.smithi033.stdout:750: expect (ObjNum 199 snap 47 seq_num 199) 2024-09-21T09:53:34.701 INFO:tasks.rados.rados.0.smithi033.stdout:751: finishing write tid 1 to smithi03385152-31 2024-09-21T09:53:34.701 INFO:tasks.rados.rados.0.smithi033.stdout:751: finishing write tid 2 to smithi03385152-31 2024-09-21T09:53:34.701 INFO:tasks.rados.rados.0.smithi033.stdout:753: finishing rollback tid 1 to smithi03385152-10 2024-09-21T09:53:34.701 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 0 (ObjNum 65 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:53:34.701 INFO:tasks.rados.rados.0.smithi033.stdout:751: finishing write tid 3 to smithi03385152-31 2024-09-21T09:53:34.701 INFO:tasks.rados.rados.0.smithi033.stdout:751: finishing write tid 4 to smithi03385152-31 2024-09-21T09:53:34.701 INFO:tasks.rados.rados.0.smithi033.stdout:750: done (6 left) 2024-09-21T09:53:34.701 INFO:tasks.rados.rados.0.smithi033.stdout:753: done (5 left) 2024-09-21T09:53:34.702 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 66 2024-09-21T09:53:34.702 INFO:tasks.rados.rados.0.smithi033.stdout:755: finishing rollback tid 0 to smithi03385152-18 2024-09-21T09:53:34.702 INFO:tasks.rados.rados.0.smithi033.stdout:756: finishing rollback tid 0 to smithi03385152-21 2024-09-21T09:53:34.702 INFO:tasks.rados.rados.0.smithi033.stdout:756: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:53:34.702 INFO:tasks.rados.rados.0.smithi033.stdout:751: finishing write tid 5 to smithi03385152-31 2024-09-21T09:53:34.702 INFO:tasks.rados.rados.0.smithi033.stdout:757: seq_num 260 ranges {1744896=761856,2506752=417792,2924544=786432} 2024-09-21T09:53:34.714 INFO:tasks.rados.rados.0.smithi033.stdout:757: writing smithi03385152-4 from 1744896 to 2506752 tid 1 2024-09-21T09:53:34.716 INFO:tasks.rados.rados.0.smithi033.stdout:757: writing smithi03385152-4 from 2506752 to 2924544 tid 2 2024-09-21T09:53:34.721 INFO:tasks.rados.rados.0.smithi033.stdout:757: writing smithi03385152-4 from 2924544 to 3710976 tid 3 2024-09-21T09:53:34.721 INFO:tasks.rados.rados.0.smithi033.stdout:758: copy_from oid 32 from oid 43 current snap is 66 2024-09-21T09:53:34.721 INFO:tasks.rados.rados.0.smithi033.stdout:755: finishing rollback tid 1 to smithi03385152-18 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:755: finishing rollback tid 2 to smithi03385152-18 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 119 (ObjNum 138 snap 32 seq_num 138) dirty exists 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:756: finishing rollback tid 2 to smithi03385152-21 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 229 (ObjNum 112 snap 25 seq_num 112) dirty exists 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:751: finishing write tid 6 to smithi03385152-31 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 510 (ObjNum 259 snap 66 seq_num 259) dirty exists 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:751: left oid 31 (ObjNum 259 snap 66 seq_num 259) 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:751: done (6 left) 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:755: done (5 left) 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:756: done (4 left) 2024-09-21T09:53:34.722 INFO:tasks.rados.rados.0.smithi033.stdout:759: snap_remove snap 64 2024-09-21T09:53:34.765 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:53:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:34 smithi033 ceph-mon[39045]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:53:34.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:34 smithi033 ceph-mon[27969]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:53:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:34 smithi105 ceph-mon[31286]: osdmap e198: 8 total, 8 up, 8 in 2024-09-21T09:53:35.030 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:34 smithi105 bash[76448]: Getting image source signatures 2024-09-21T09:53:35.030 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:34 smithi105 bash[76448]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-21T09:53:35.030 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:34 smithi105 bash[76448]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-21T09:53:35.031 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:34 smithi105 bash[76448]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-21T09:53:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:53:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 starting - - - - 2024-09-21T09:53:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 starting - - - - 2024-09-21T09:53:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (6m) 42s ago 6m 14.4M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:53:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (6m) 41s ago 6m 18.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:53:35.341 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283 running (67s) 41s ago 9m 497M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b10e03ec7c52 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:9283,8765 running (11m) 42s ago 11m 301M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 21061eedc154 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (11m) 42s ago 11m 56.0M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 220cb0bc67bb 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (10m) 41s ago 10m 46.1M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c2741af3f853 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (10m) 42s ago 10m 45.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f8b68f40cc5 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 starting - - - - 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (4m) 41s ago 4m 16.9M - 1.5.0 0da6a335fe13 446328798f33 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (9m) 42s ago 9m 98.4M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (9m) 42s ago 9m 105M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (8m) 42s ago 8m 83.7M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (8m) 42s ago 8m 68.3M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (7m) 41s ago 7m 98.8M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (7m) 41s ago 7m 100M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:53:35.342 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (7m) 41s ago 7m 66.8M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:53:35.343 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (6m) 41s ago 6m 78.4M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:53:35.343 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 starting - - - - 2024-09-21T09:53:35.577 INFO:tasks.rados.rados.0.smithi033.stdout:759: done (4 left) 2024-09-21T09:53:35.578 INFO:tasks.rados.rados.0.smithi033.stdout:760: snap_remove snap 59 2024-09-21T09:53:35.701 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:53:35.701 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:53:35.701 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:53:35.701 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:53:35.701 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:53:35.701 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 1, 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 14, 2024-09-21T09:53:35.702 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:53:35.703 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:53:35.703 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:53:35.703 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:35 smithi033 ceph-mon[27969]: pgmap v75: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.0 MiB/s wr, 16 op/s 2024-09-21T09:53:35.703 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:35 smithi033 ceph-mon[27969]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:53:35.703 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:35 smithi033 ceph-mon[27969]: from='client.25031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:35.703 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:35 smithi033 ceph-mon[39045]: pgmap v75: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.0 MiB/s wr, 16 op/s 2024-09-21T09:53:35.703 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:35 smithi033 ceph-mon[39045]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:53:35.704 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:35 smithi033 ceph-mon[39045]: from='client.25031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:36.000 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:53:36.001 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:53:36.001 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:53:36.001 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:53:36.001 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [], 2024-09-21T09:53:36.001 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "1/20 daemons upgraded", 2024-09-21T09:53:36.001 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "", 2024-09-21T09:53:36.001 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:53:36.001 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:53:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:35 smithi105 ceph-mon[31286]: pgmap v75: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.0 MiB/s wr, 16 op/s 2024-09-21T09:53:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:35 smithi105 ceph-mon[31286]: osdmap e199: 8 total, 8 up, 8 in 2024-09-21T09:53:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:35 smithi105 ceph-mon[31286]: from='client.25031 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:36.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:36.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:36.492 INFO:tasks.rados.rados.0.smithi033.stdout:760: done (4 left) 2024-09-21T09:53:36.492 INFO:tasks.rados.rados.0.smithi033.stdout:761: copy_from oid 15 from oid 11 current snap is 66 2024-09-21T09:53:36.492 INFO:tasks.rados.rados.0.smithi033.stdout:752: expect (ObjNum 234 snap 55 seq_num 234) 2024-09-21T09:53:36.530 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 bash[76448]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-21T09:53:36.530 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 bash[76448]: Writing manifest to image destination 2024-09-21T09:53:36.600 INFO:tasks.rados.rados.0.smithi033.stdout:754: expect (ObjNum 177 snap 40 seq_num 177) 2024-09-21T09:53:36.801 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-mon[31286]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:36.801 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-mon[31286]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:36.801 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-mon[31286]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:53:36.801 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2521469004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:36.801 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-mon[31286]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:53:36.801 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 podman[76448]: 2024-09-21 09:53:36.53236342 +0000 UTC m=+2.414501982 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-21T09:53:36.801 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 podman[76448]: 2024-09-21 09:53:36.657167703 +0000 UTC m=+2.539306304 container create c79287d80715ac184038d94d33f856407597b1b8d3c82429cb0c0b2cc20f0e18 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:53:36.825 INFO:tasks.rados.rados.0.smithi033.stdout:757: finishing write tid 1 to smithi03385152-4 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:757: finishing write tid 2 to smithi03385152-4 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:757: finishing write tid 3 to smithi03385152-4 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:757: finishing write tid 4 to smithi03385152-4 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:757: finishing write tid 5 to smithi03385152-4 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 116 (ObjNum 260 snap 66 seq_num 260) dirty exists 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:757: left oid 4 (ObjNum 260 snap 66 seq_num 260) 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:758: finishing copy_from racing read to smithi03385152-32 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:758: finishing copy_from to smithi03385152-32 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 61 (ObjNum 199 snap 47 seq_num 199) dirty exists 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:761: finishing copy_from racing read to smithi03385152-15 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:761: finishing copy_from to smithi03385152-15 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 62 (ObjNum 192 snap 45 seq_num 192) dirty exists 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:752: done (4 left) 2024-09-21T09:53:36.826 INFO:tasks.rados.rados.0.smithi033.stdout:754: done (3 left) 2024-09-21T09:53:36.827 INFO:tasks.rados.rados.0.smithi033.stdout:757: done (2 left) 2024-09-21T09:53:36.827 INFO:tasks.rados.rados.0.smithi033.stdout:758: done (1 left) 2024-09-21T09:53:36.827 INFO:tasks.rados.rados.0.smithi033.stdout:761: done (0 left) 2024-09-21T09:53:36.827 INFO:tasks.rados.rados.0.smithi033.stdout:762: setattr oid 14 current snap is 66 2024-09-21T09:53:36.827 INFO:tasks.rados.rados.0.smithi033.stdout:763: setattr oid 46 current snap is 66 2024-09-21T09:53:36.827 INFO:tasks.rados.rados.0.smithi033.stdout:764: snap_create 2024-09-21T09:53:36.829 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 484 (ObjNum 186 snap 44 seq_num 186) dirty exists 2024-09-21T09:53:36.829 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 485 (ObjNum 33 snap 0 seq_num 90775552) dirty exists 2024-09-21T09:53:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[39045]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[39045]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[39045]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:53:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2521469004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[39045]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:53:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[27969]: from='client.15129 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:36.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[27969]: from='client.15135 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:36.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[27969]: osdmap e200: 8 total, 8 up, 8 in 2024-09-21T09:53:36.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/2521469004' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:53:36.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:36 smithi033 ceph-mon[27969]: osdmap e201: 8 total, 8 up, 8 in 2024-09-21T09:53:37.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:37.280 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 podman[76448]: 2024-09-21 09:53:36.934991106 +0000 UTC m=+2.817129666 container init c79287d80715ac184038d94d33f856407597b1b8d3c82429cb0c0b2cc20f0e18 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 podman[76448]: 2024-09-21 09:53:36.940342493 +0000 UTC m=+2.822481052 container start c79287d80715ac184038d94d33f856407597b1b8d3c82429cb0c0b2cc20f0e18 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.943Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.943Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.943Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.943Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=arp 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-21T09:53:37.281 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=edac 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-21T09:53:37.282 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=os 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-21T09:53:37.283 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-21T09:53:37.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-21T09:53:37.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-21T09:53:37.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-21T09:53:37.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-21T09:53:37.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=stat 2024-09-21T09:53:37.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-21T09:53:37.284 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=time 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=uname 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-21T09:53:37.285 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b[76523]: ts=2024-09-21T09:53:36.944Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-21T09:53:37.286 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:37 smithi105 bash[76448]: c79287d80715ac184038d94d33f856407597b1b8d3c82429cb0c0b2cc20f0e18 2024-09-21T09:53:37.286 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 09:53:37 smithi105 systemd[1]: Started Ceph node-exporter.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:37.589 INFO:tasks.rados.rados.0.smithi033.stdout:762: done (2 left) 2024-09-21T09:53:37.590 INFO:tasks.rados.rados.0.smithi033.stdout:763: done (1 left) 2024-09-21T09:53:37.590 INFO:tasks.rados.rados.0.smithi033.stdout:764: done (0 left) 2024-09-21T09:53:37.590 INFO:tasks.rados.rados.0.smithi033.stdout:765: rollback oid 27 current snap is 67 2024-09-21T09:53:37.590 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 27 to 45 2024-09-21T09:53:37.590 INFO:tasks.rados.rados.0.smithi033.stdout:766: snap_remove snap 65 2024-09-21T09:53:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[39045]: from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[39045]: pgmap v78: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.0 MiB/s wr, 16 op/s 2024-09-21T09:53:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:53:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[27969]: from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[27969]: pgmap v78: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.0 MiB/s wr, 16 op/s 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:53:37.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:37 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:37 smithi105 ceph-mon[31286]: from='client.25055 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:53:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:37 smithi105 ceph-mon[31286]: pgmap v78: 121 pgs: 2 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 116 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.0 MiB/s wr, 16 op/s 2024-09-21T09:53:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:37 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:37 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:38.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:37 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:53:38.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:37 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:38.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:37 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:53:38.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:37 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:38.597 INFO:tasks.rados.rados.0.smithi033.stdout:766: done (1 left) 2024-09-21T09:53:38.597 INFO:tasks.rados.rados.0.smithi033.stdout:767: copy_from oid 11 from oid 41 current snap is 67 2024-09-21T09:53:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:765: finishing rollback tid 0 to smithi03385152-27 2024-09-21T09:53:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:765: finishing rollback tid 1 to smithi03385152-27 2024-09-21T09:53:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:765: finishing rollback tid 2 to smithi03385152-27 2024-09-21T09:53:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 270 (ObjNum 191 snap 45 seq_num 191) dirty exists 2024-09-21T09:53:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:765: done (1 left) 2024-09-21T09:53:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:768: copy_from oid 46 from oid 17 current snap is 67 2024-09-21T09:53:38.598 INFO:tasks.rados.rados.0.smithi033.stdout:769: delete oid 29 current snap is 67 2024-09-21T09:53:38.603 INFO:tasks.rados.rados.0.smithi033.stdout:769: done (2 left) 2024-09-21T09:53:38.603 INFO:tasks.rados.rados.0.smithi033.stdout:770: snap_remove snap 56 2024-09-21T09:53:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:53:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:53:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:38.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:38.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:53:38.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:38.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:38.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:38.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:38.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:38 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-21T09:53:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:53:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T09:53:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-21T09:53:39.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: osdmap e202: 8 total, 8 up, 8 in 2024-09-21T09:53:39.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:39.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:39.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:53:39.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:53:39.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:38 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:39.613 INFO:tasks.rados.rados.0.smithi033.stdout:770: done (2 left) 2024-09-21T09:53:39.614 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 67 2024-09-21T09:53:39.614 INFO:tasks.rados.rados.0.smithi033.stdout:771: seq_num 262 ranges {3612672=794624,4407296=589824,4997120=614400} 2024-09-21T09:53:39.637 INFO:tasks.rados.rados.0.smithi033.stdout:771: writing smithi03385152-12 from 3612672 to 4407296 tid 1 2024-09-21T09:53:39.641 INFO:tasks.rados.rados.0.smithi033.stdout:771: writing smithi03385152-12 from 4407296 to 4997120 tid 2 2024-09-21T09:53:39.644 INFO:tasks.rados.rados.0.smithi033.stdout:771: writing smithi03385152-12 from 4997120 to 5611520 tid 3 2024-09-21T09:53:39.644 INFO:tasks.rados.rados.0.smithi033.stdout:772: delete oid 22 current snap is 67 2024-09-21T09:53:39.645 INFO:tasks.rados.rados.0.smithi033.stdout:768: finishing copy_from racing read to smithi03385152-46 2024-09-21T09:53:39.645 INFO:tasks.rados.rados.0.smithi033.stdout:767: finishing copy_from racing read to smithi03385152-11 2024-09-21T09:53:39.645 INFO:tasks.rados.rados.0.smithi033.stdout:768: finishing copy_from to smithi03385152-46 2024-09-21T09:53:39.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 347 (ObjNum 253 snap 65 seq_num 253) dirty exists 2024-09-21T09:53:39.645 INFO:tasks.rados.rados.0.smithi033.stdout:767: finishing copy_from to smithi03385152-11 2024-09-21T09:53:39.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 346 (ObjNum 242 snap 63 seq_num 242) dirty exists 2024-09-21T09:53:39.645 INFO:tasks.rados.rados.0.smithi033.stdout:771: finishing write tid 1 to smithi03385152-12 2024-09-21T09:53:39.645 INFO:tasks.rados.rados.0.smithi033.stdout:771: finishing write tid 2 to smithi03385152-12 2024-09-21T09:53:39.646 INFO:tasks.rados.rados.0.smithi033.stdout:767: done (3 left) 2024-09-21T09:53:39.647 INFO:tasks.rados.rados.0.smithi033.stdout:768: done (2 left) 2024-09-21T09:53:39.647 INFO:tasks.rados.rados.0.smithi033.stdout:772: done (1 left) 2024-09-21T09:53:39.647 INFO:tasks.rados.rados.0.smithi033.stdout:773: snap_remove snap 55 2024-09-21T09:53:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:39 smithi033 ceph-mon[39045]: pgmap v81: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:53:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:39 smithi033 ceph-mon[39045]: Upgrade: Updating mgr.y 2024-09-21T09:53:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:39 smithi033 ceph-mon[39045]: Deploying daemon mgr.y on smithi033 2024-09-21T09:53:39.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:39 smithi033 ceph-mon[39045]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:53:39.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:39 smithi033 ceph-mon[27969]: pgmap v81: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:53:39.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:39 smithi033 ceph-mon[27969]: Upgrade: Updating mgr.y 2024-09-21T09:53:39.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:39 smithi033 ceph-mon[27969]: Deploying daemon mgr.y on smithi033 2024-09-21T09:53:39.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:39 smithi033 ceph-mon[27969]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:53:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:39 smithi105 ceph-mon[31286]: pgmap v81: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:53:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:39 smithi105 ceph-mon[31286]: Upgrade: Updating mgr.y 2024-09-21T09:53:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:39 smithi105 ceph-mon[31286]: Deploying daemon mgr.y on smithi033 2024-09-21T09:53:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:39 smithi105 ceph-mon[31286]: osdmap e203: 8 total, 8 up, 8 in 2024-09-21T09:53:40.608 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:40 smithi033 systemd[1]: Stopping Ceph mgr.y for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:40.615 INFO:tasks.rados.rados.0.smithi033.stdout:773: done (1 left) 2024-09-21T09:53:40.615 INFO:tasks.rados.rados.0.smithi033.stdout:774: read oid 9 snap -1 2024-09-21T09:53:40.615 INFO:tasks.rados.rados.0.smithi033.stdout:774: expect deleted 2024-09-21T09:53:40.615 INFO:tasks.rados.rados.0.smithi033.stdout:775: snap_remove snap 45 2024-09-21T09:53:40.905 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:40 smithi033 ceph-mon[39045]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:53:40.906 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:40 smithi033 podman[88385]: 2024-09-21 09:53:40.612656299 +0000 UTC m=+0.172631237 container died 21061eedc1543bd6187bb850b6fee9b5edfd848ba05dbceec926b4325b838093 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-21T09:53:40.906 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:40 smithi033 ceph-mon[27969]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:53:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:40 smithi105 ceph-mon[31286]: osdmap e204: 8 total, 8 up, 8 in 2024-09-21T09:53:41.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:40 smithi033 podman[88385]: 2024-09-21 09:53:40.978461983 +0000 UTC m=+0.538436900 container cleanup 21061eedc1543bd6187bb850b6fee9b5edfd848ba05dbceec926b4325b838093 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:53:41.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:40 smithi033 bash[88385]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y 2024-09-21T09:53:41.493 INFO:tasks.rados.rados.0.smithi033.stdout:775: done (2 left) 2024-09-21T09:53:41.493 INFO:tasks.rados.rados.0.smithi033.stdout:776: read oid 47 snap -1 2024-09-21T09:53:41.493 INFO:tasks.rados.rados.0.smithi033.stdout:776: expect (ObjNum 166 snap 37 seq_num 166) 2024-09-21T09:53:41.493 INFO:tasks.rados.rados.0.smithi033.stdout:777: rollback oid 33 current snap is 67 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 66 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:771: finishing write tid 3 to smithi03385152-12 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:771: finishing write tid 4 to smithi03385152-12 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:771: finishing write tid 5 to smithi03385152-12 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 355 (ObjNum 262 snap 67 seq_num 262) dirty exists 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:771: left oid 12 (ObjNum 262 snap 67 seq_num 262) 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:771: done (3 left) 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:774: done (2 left) 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 67 2024-09-21T09:53:41.494 INFO:tasks.rados.rados.0.smithi033.stdout:778: seq_num 264 ranges {2252800=425984,2678784=761856,3440640=770048,4210688=73728} 2024-09-21T09:53:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:778: writing smithi03385152-43 from 2252800 to 2678784 tid 1 2024-09-21T09:53:41.517 INFO:tasks.rados.rados.0.smithi033.stdout:778: writing smithi03385152-43 from 2678784 to 3440640 tid 2 2024-09-21T09:53:41.523 INFO:tasks.rados.rados.0.smithi033.stdout:778: writing smithi03385152-43 from 3440640 to 4210688 tid 3 2024-09-21T09:53:41.523 INFO:tasks.rados.rados.0.smithi033.stdout:778: writing smithi03385152-43 from 4210688 to 4284416 tid 4 2024-09-21T09:53:41.524 INFO:tasks.rados.rados.0.smithi033.stdout:779: read oid 24 snap -1 2024-09-21T09:53:41.525 INFO:tasks.rados.rados.0.smithi033.stdout:779: expect (ObjNum 251 snap 65 seq_num 251) 2024-09-21T09:53:41.525 INFO:tasks.rados.rados.0.smithi033.stdout:777: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:53:41.525 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 244 (ObjNum 369 snap 0 seq_num 91136192) dirty dne 2024-09-21T09:53:41.525 INFO:tasks.rados.rados.0.smithi033.stdout:778: finishing write tid 1 to smithi03385152-43 2024-09-21T09:53:41.525 INFO:tasks.rados.rados.0.smithi033.stdout:778: finishing write tid 2 to smithi03385152-43 2024-09-21T09:53:41.525 INFO:tasks.rados.rados.0.smithi033.stdout:777: done (3 left) 2024-09-21T09:53:41.525 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 67 2024-09-21T09:53:41.525 INFO:tasks.rados.rados.0.smithi033.stdout:780: seq_num 265 ranges {0=442368,442368=647168,1089536=573440,1662976=385024} 2024-09-21T09:53:41.526 INFO:tasks.rados.rados.0.smithi033.stdout:780: writing smithi03385152-19 from 0 to 442368 tid 1 2024-09-21T09:53:41.531 INFO:tasks.rados.rados.0.smithi033.stdout:780: writing smithi03385152-19 from 442368 to 1089536 tid 2 2024-09-21T09:53:41.536 INFO:tasks.rados.rados.0.smithi033.stdout:780: writing smithi03385152-19 from 1089536 to 1662976 tid 3 2024-09-21T09:53:41.539 INFO:tasks.rados.rados.0.smithi033.stdout:780: writing smithi03385152-19 from 1662976 to 2048000 tid 4 2024-09-21T09:53:41.539 INFO:tasks.rados.rados.0.smithi033.stdout:781: rollback oid 50 current snap is 67 2024-09-21T09:53:41.539 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 50 to 66 2024-09-21T09:53:41.539 INFO:tasks.rados.rados.0.smithi033.stdout:776: expect (ObjNum 166 snap 37 seq_num 166) 2024-09-21T09:53:41.609 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:41 smithi033 podman[88404]: 2024-09-21 09:53:41.330934338 +0000 UTC m=+0.722470379 container remove 21061eedc1543bd6187bb850b6fee9b5edfd848ba05dbceec926b4325b838093 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T09:53:41.610 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:41 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:53:41.610 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:41 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y.service: Failed with result 'exit-code'. 2024-09-21T09:53:41.610 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:41 smithi033 systemd[1]: Stopped Ceph mgr.y for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:41.610 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:41 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y.service: Consumed 45.782s CPU time. 2024-09-21T09:53:41.610 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:41 smithi033 ceph-mon[27969]: pgmap v84: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:53:41.610 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:41 smithi033 ceph-mon[27969]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:53:41.610 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:41 smithi033 ceph-mon[27969]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:53:41.863 INFO:tasks.rados.rados.0.smithi033.stdout:778: finishing write tid 3 to smithi03385152-43 2024-09-21T09:53:41.863 INFO:tasks.rados.rados.0.smithi033.stdout:778: finishing write tid 4 to smithi03385152-43 2024-09-21T09:53:41.863 INFO:tasks.rados.rados.0.smithi033.stdout:778: finishing write tid 5 to smithi03385152-43 2024-09-21T09:53:41.863 INFO:tasks.rados.rados.0.smithi033.stdout:778: finishing write tid 6 to smithi03385152-43 2024-09-21T09:53:41.863 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 351 (ObjNum 264 snap 67 seq_num 264) dirty exists 2024-09-21T09:53:41.863 INFO:tasks.rados.rados.0.smithi033.stdout:778: left oid 43 (ObjNum 264 snap 67 seq_num 264) 2024-09-21T09:53:41.863 INFO:tasks.rados.rados.0.smithi033.stdout:776: done (4 left) 2024-09-21T09:53:41.864 INFO:tasks.rados.rados.0.smithi033.stdout:778: done (3 left) 2024-09-21T09:53:41.864 INFO:tasks.rados.rados.0.smithi033.stdout:782: read oid 3 snap -1 2024-09-21T09:53:41.864 INFO:tasks.rados.rados.0.smithi033.stdout:782: expect deleted 2024-09-21T09:53:41.864 INFO:tasks.rados.rados.0.smithi033.stdout:783: copy_from oid 45 from oid 16 current snap is 67 2024-09-21T09:53:41.864 INFO:tasks.rados.rados.0.smithi033.stdout:781: finishing rollback tid 0 to smithi03385152-50 2024-09-21T09:53:41.864 INFO:tasks.rados.rados.0.smithi033.stdout:781: finishing rollback tid 1 to smithi03385152-50 2024-09-21T09:53:41.864 INFO:tasks.rados.rados.0.smithi033.stdout:779: expect (ObjNum 251 snap 65 seq_num 251) 2024-09-21T09:53:41.931 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:41 smithi033 systemd[1]: Starting Ceph mgr.y for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:53:41.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:41 smithi033 ceph-mon[39045]: pgmap v84: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:53:41.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:41 smithi033 ceph-mon[39045]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:53:41.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:41 smithi033 ceph-mon[39045]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:53:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:41 smithi105 ceph-mon[31286]: pgmap v84: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 25 op/s 2024-09-21T09:53:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:41 smithi105 ceph-mon[31286]: osdmap e205: 8 total, 8 up, 8 in 2024-09-21T09:53:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:41 smithi105 ceph-mon[31286]: osdmap e206: 8 total, 8 up, 8 in 2024-09-21T09:53:42.030 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:53:41] "GET /metrics HTTP/1.1" 200 36212 "" "Prometheus/2.51.0" 2024-09-21T09:53:42.058 INFO:tasks.rados.rados.0.smithi033.stdout:780: finishing write tid 1 to smithi03385152-19 2024-09-21T09:53:42.059 INFO:tasks.rados.rados.0.smithi033.stdout:780: finishing write tid 2 to smithi03385152-19 2024-09-21T09:53:42.059 INFO:tasks.rados.rados.0.smithi033.stdout:781: finishing rollback tid 2 to smithi03385152-50 2024-09-21T09:53:42.059 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 156 (ObjNum 224 snap 52 seq_num 224) dirty exists 2024-09-21T09:53:42.059 INFO:tasks.rados.rados.0.smithi033.stdout:780: finishing write tid 3 to smithi03385152-19 2024-09-21T09:53:42.059 INFO:tasks.rados.rados.0.smithi033.stdout:780: finishing write tid 4 to smithi03385152-19 2024-09-21T09:53:42.059 INFO:tasks.rados.rados.0.smithi033.stdout:779: done (4 left) 2024-09-21T09:53:42.060 INFO:tasks.rados.rados.0.smithi033.stdout:781: done (3 left) 2024-09-21T09:53:42.060 INFO:tasks.rados.rados.0.smithi033.stdout:784: rmattr oid 13 current snap is 67 2024-09-21T09:53:42.060 INFO:tasks.rados.rados.0.smithi033.stdout:780: finishing write tid 5 to smithi03385152-19 2024-09-21T09:53:42.060 INFO:tasks.rados.rados.0.smithi033.stdout:780: finishing write tid 6 to smithi03385152-19 2024-09-21T09:53:42.060 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 133 (ObjNum 265 snap 67 seq_num 265) dirty exists 2024-09-21T09:53:42.060 INFO:tasks.rados.rados.0.smithi033.stdout:780: left oid 19 (ObjNum 265 snap 67 seq_num 265) 2024-09-21T09:53:42.060 INFO:tasks.rados.rados.0.smithi033.stdout:783: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:53:42.060 INFO:tasks.rados.rados.0.smithi033.stdout:783: finishing copy_from to smithi03385152-45 2024-09-21T09:53:42.061 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 479 (ObjNum 250 snap 65 seq_num 250) dirty exists 2024-09-21T09:53:42.061 INFO:tasks.rados.rados.0.smithi033.stdout:780: done (3 left) 2024-09-21T09:53:42.061 INFO:tasks.rados.rados.0.smithi033.stdout:782: done (2 left) 2024-09-21T09:53:42.061 INFO:tasks.rados.rados.0.smithi033.stdout:783: done (1 left) 2024-09-21T09:53:42.061 INFO:tasks.rados.rados.0.smithi033.stdout:785: rollback oid 11 current snap is 67 2024-09-21T09:53:42.061 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 11 to 57 2024-09-21T09:53:42.061 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 67 2024-09-21T09:53:42.061 INFO:tasks.rados.rados.0.smithi033.stdout:786: seq_num 266 ranges {4284416=458752,4743168=524288,5267456=729088,5996544=352256} 2024-09-21T09:53:42.083 INFO:tasks.rados.rados.0.smithi033.stdout:786: writing smithi03385152-43 from 4284416 to 4743168 tid 1 2024-09-21T09:53:42.086 INFO:tasks.rados.rados.0.smithi033.stdout:786: writing smithi03385152-43 from 4743168 to 5267456 tid 2 2024-09-21T09:53:42.090 INFO:tasks.rados.rados.0.smithi033.stdout:786: writing smithi03385152-43 from 5267456 to 5996544 tid 3 2024-09-21T09:53:42.093 INFO:tasks.rados.rados.0.smithi033.stdout:786: writing smithi03385152-43 from 5996544 to 6348800 tid 4 2024-09-21T09:53:42.093 INFO:tasks.rados.rados.0.smithi033.stdout:787: read oid 46 snap -1 2024-09-21T09:53:42.094 INFO:tasks.rados.rados.0.smithi033.stdout:787: expect (ObjNum 253 snap 65 seq_num 253) 2024-09-21T09:53:42.094 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 67 2024-09-21T09:53:42.094 INFO:tasks.rados.rados.0.smithi033.stdout:785: finishing rollback tid 0 to smithi03385152-11 2024-09-21T09:53:42.094 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 552 (ObjNum 248 snap 65 seq_num 248) dirty exists 2024-09-21T09:53:42.094 INFO:tasks.rados.rados.0.smithi033.stdout:785: finishing rollback tid 1 to smithi03385152-11 2024-09-21T09:53:42.095 INFO:tasks.rados.rados.0.smithi033.stdout:785: finishing rollback tid 2 to smithi03385152-11 2024-09-21T09:53:42.095 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 354 (ObjNum 192 snap 45 seq_num 192) dirty exists 2024-09-21T09:53:42.095 INFO:tasks.rados.rados.0.smithi033.stdout:786: finishing write tid 1 to smithi03385152-43 2024-09-21T09:53:42.095 INFO:tasks.rados.rados.0.smithi033.stdout:786: finishing write tid 2 to smithi03385152-43 2024-09-21T09:53:42.095 INFO:tasks.rados.rados.0.smithi033.stdout:788: seq_num 267 ranges {1441792=475136,1916928=458752,2375680=540672,2916352=606208} 2024-09-21T09:53:42.106 INFO:tasks.rados.rados.0.smithi033.stdout:788: writing smithi03385152-30 from 1441792 to 1916928 tid 1 2024-09-21T09:53:42.112 INFO:tasks.rados.rados.0.smithi033.stdout:788: writing smithi03385152-30 from 1916928 to 2375680 tid 2 2024-09-21T09:53:42.115 INFO:tasks.rados.rados.0.smithi033.stdout:788: writing smithi03385152-30 from 2375680 to 2916352 tid 3 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:788: writing smithi03385152-30 from 2916352 to 3522560 tid 4 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:784: done (4 left) 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:785: done (3 left) 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:789: setattr oid 26 current snap is 67 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:786: finishing write tid 3 to smithi03385152-43 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:786: finishing write tid 4 to smithi03385152-43 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:786: finishing write tid 5 to smithi03385152-43 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:786: finishing write tid 6 to smithi03385152-43 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 359 (ObjNum 266 snap 67 seq_num 266) dirty exists 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:786: left oid 43 (ObjNum 266 snap 67 seq_num 266) 2024-09-21T09:53:42.118 INFO:tasks.rados.rados.0.smithi033.stdout:787: expect (ObjNum 253 snap 65 seq_num 253) 2024-09-21T09:53:42.166 INFO:tasks.rados.rados.0.smithi033.stdout:788: finishing write tid 1 to smithi03385152-30 2024-09-21T09:53:42.166 INFO:tasks.rados.rados.0.smithi033.stdout:788: finishing write tid 2 to smithi03385152-30 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:788: finishing write tid 3 to smithi03385152-30 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:788: finishing write tid 4 to smithi03385152-30 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:788: finishing write tid 5 to smithi03385152-30 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:788: finishing write tid 6 to smithi03385152-30 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 558 (ObjNum 267 snap 67 seq_num 267) dirty exists 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:788: left oid 30 (ObjNum 267 snap 67 seq_num 267) 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:786: done (3 left) 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:787: done (2 left) 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:788: done (1 left) 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:790: read oid 21 snap -1 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:790: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:53:42.167 INFO:tasks.rados.rados.0.smithi033.stdout:791: snap_remove snap 61 2024-09-21T09:53:42.186 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:41 smithi033 podman[88535]: 2024-09-21 09:53:41.831637741 +0000 UTC m=+0.016480118 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:53:42.186 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:41 smithi033 podman[88535]: 2024-09-21 09:53:41.940345623 +0000 UTC m=+0.125187998 container create 917c670c2752529c94adb0dd03344b77d50e24316c202e543fee1916845e2287 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, ceph=True) 2024-09-21T09:53:42.186 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:42 smithi033 podman[88535]: 2024-09-21 09:53:42.15137875 +0000 UTC m=+0.336221137 container init 917c670c2752529c94adb0dd03344b77d50e24316c202e543fee1916845e2287 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:53:42.187 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:42 smithi033 podman[88535]: 2024-09-21 09:53:42.156423986 +0000 UTC m=+0.341266374 container start 917c670c2752529c94adb0dd03344b77d50e24316c202e543fee1916845e2287 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T09:53:42.498 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:42 smithi033 ceph-mgr[88573]: -- 172.21.15.33:0/1752740993 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b040b3b0e0 con 0x55b040b18800 2024-09-21T09:53:42.498 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:42 smithi033 bash[88535]: 917c670c2752529c94adb0dd03344b77d50e24316c202e543fee1916845e2287 2024-09-21T09:53:42.498 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:42 smithi033 systemd[1]: Started Ceph mgr.y for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:53:42.616 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 356 (ObjNum 33 snap 0 seq_num 90719392) dirty exists 2024-09-21T09:53:42.616 INFO:tasks.rados.rados.0.smithi033.stdout:790: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:53:42.759 INFO:tasks.rados.rados.0.smithi033.stdout:789: done (2 left) 2024-09-21T09:53:42.759 INFO:tasks.rados.rados.0.smithi033.stdout:790: done (1 left) 2024-09-21T09:53:42.759 INFO:tasks.rados.rados.0.smithi033.stdout:791: done (0 left) 2024-09-21T09:53:42.759 INFO:tasks.rados.rados.0.smithi033.stdout:792: snap_remove snap 57 2024-09-21T09:53:43.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:43 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:43 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:43 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:43.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:43 smithi105 ceph-mon[31286]: pgmap v87: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:43.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:43 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:43 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:43 smithi105 ceph-mon[31286]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:53:43.617 INFO:tasks.rados.rados.0.smithi033.stdout:792: done (0 left) 2024-09-21T09:53:43.617 INFO:tasks.rados.rados.0.smithi033.stdout:793: snap_remove snap 66 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[39045]: pgmap v87: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[39045]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.665 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.666 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:43.666 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[27969]: pgmap v87: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:43.666 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.666 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:43.666 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:43 smithi033 ceph-mon[27969]: osdmap e207: 8 total, 8 up, 8 in 2024-09-21T09:53:44.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:44.146Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:44.623 INFO:tasks.rados.rados.0.smithi033.stdout:793: done (0 left) 2024-09-21T09:53:44.623 INFO:tasks.rados.rados.0.smithi033.stdout:794: snap_create 2024-09-21T09:53:44.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:44 smithi105 ceph-mon[31286]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:53:44.939 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:44 smithi033 ceph-mon[39045]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:53:44.939 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:44 smithi033 ceph-mon[27969]: osdmap e208: 8 total, 8 up, 8 in 2024-09-21T09:53:45.622 INFO:tasks.rados.rados.0.smithi033.stdout:794: done (0 left) 2024-09-21T09:53:45.623 INFO:tasks.rados.rados.0.smithi033.stdout:795: rollback oid 21 current snap is 68 2024-09-21T09:53:45.623 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 67 2024-09-21T09:53:45.623 INFO:tasks.rados.rados.0.smithi033.stdout:796: copy_from oid 28 from oid 37 current snap is 68 2024-09-21T09:53:45.623 INFO:tasks.rados.rados.0.smithi033.stdout:797: delete oid 1 current snap is 68 2024-09-21T09:53:45.625 INFO:tasks.rados.rados.0.smithi033.stdout:797: done (2 left) 2024-09-21T09:53:45.625 INFO:tasks.rados.rados.0.smithi033.stdout:798: delete oid 20 current snap is 68 2024-09-21T09:53:45.630 INFO:tasks.rados.rados.0.smithi033.stdout:795: finishing rollback tid 0 to smithi03385152-21 2024-09-21T09:53:45.630 INFO:tasks.rados.rados.0.smithi033.stdout:795: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:53:45.631 INFO:tasks.rados.rados.0.smithi033.stdout:795: finishing rollback tid 2 to smithi03385152-21 2024-09-21T09:53:45.631 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 251 (ObjNum 112 snap 25 seq_num 112) dirty exists 2024-09-21T09:53:45.631 INFO:tasks.rados.rados.0.smithi033.stdout:795: done (2 left) 2024-09-21T09:53:45.631 INFO:tasks.rados.rados.0.smithi033.stdout:798: done (1 left) 2024-09-21T09:53:45.631 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 68 2024-09-21T09:53:45.631 INFO:tasks.rados.rados.0.smithi033.stdout:799: seq_num 270 ranges {4644864=524288,5169152=516096,5685248=761856,6447104=327680} 2024-09-21T09:53:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:799: writing smithi03385152-24 from 4644864 to 5169152 tid 1 2024-09-21T09:53:45.663 INFO:tasks.rados.rados.0.smithi033.stdout:799: writing smithi03385152-24 from 5169152 to 5685248 tid 2 2024-09-21T09:53:45.667 INFO:tasks.rados.rados.0.smithi033.stdout:799: writing smithi03385152-24 from 5685248 to 6447104 tid 3 2024-09-21T09:53:45.669 INFO:tasks.rados.rados.0.smithi033.stdout:799: writing smithi03385152-24 from 6447104 to 6774784 tid 4 2024-09-21T09:53:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:800: rmattr oid 2 current snap is 68 2024-09-21T09:53:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:800: done (2 left) 2024-09-21T09:53:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:801: snap_create 2024-09-21T09:53:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:796: finishing copy_from racing read to smithi03385152-28 2024-09-21T09:53:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:796: finishing copy_from to smithi03385152-28 2024-09-21T09:53:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 145 (ObjNum 174 snap 38 seq_num 174) dirty exists 2024-09-21T09:53:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:799: finishing write tid 1 to smithi03385152-24 2024-09-21T09:53:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:799: finishing write tid 2 to smithi03385152-24 2024-09-21T09:53:45.671 INFO:tasks.rados.rados.0.smithi033.stdout:799: finishing write tid 3 to smithi03385152-24 2024-09-21T09:53:45.671 INFO:tasks.rados.rados.0.smithi033.stdout:799: finishing write tid 4 to smithi03385152-24 2024-09-21T09:53:45.671 INFO:tasks.rados.rados.0.smithi033.stdout:799: finishing write tid 5 to smithi03385152-24 2024-09-21T09:53:45.672 INFO:tasks.rados.rados.0.smithi033.stdout:799: finishing write tid 6 to smithi03385152-24 2024-09-21T09:53:45.672 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 149 (ObjNum 270 snap 68 seq_num 270) dirty exists 2024-09-21T09:53:45.672 INFO:tasks.rados.rados.0.smithi033.stdout:799: left oid 24 (ObjNum 270 snap 68 seq_num 270) 2024-09-21T09:53:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:45 smithi033 ceph-mon[39045]: pgmap v90: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-21T09:53:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:45 smithi033 ceph-mon[39045]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:53:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:45 smithi033 ceph-mon[27969]: pgmap v90: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-21T09:53:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:45 smithi033 ceph-mon[27969]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:53:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:45 smithi105 ceph-mon[31286]: pgmap v90: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-21T09:53:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:45 smithi105 ceph-mon[31286]: osdmap e209: 8 total, 8 up, 8 in 2024-09-21T09:53:46.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:46.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:46.493 INFO:tasks.rados.rados.0.smithi033.stdout:796: done (2 left) 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:799: done (1 left) 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:801: done (0 left) 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:802: rmattr oid 35 current snap is 69 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:802: done (0 left) 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:803: read oid 27 snap -1 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:803: expect (ObjNum 191 snap 45 seq_num 191) 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:804: read oid 37 snap -1 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:804: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:805: read oid 5 snap -1 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:805: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:806: rollback oid 8 current snap is 69 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 8 to 68 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 69 2024-09-21T09:53:46.494 INFO:tasks.rados.rados.0.smithi033.stdout:807: seq_num 271 ranges {5660672=540672,6201344=802816,7004160=516096,7520256=286720} 2024-09-21T09:53:46.530 INFO:tasks.rados.rados.0.smithi033.stdout:807: writing smithi03385152-47 from 5660672 to 6201344 tid 1 2024-09-21T09:53:46.535 INFO:tasks.rados.rados.0.smithi033.stdout:807: writing smithi03385152-47 from 6201344 to 7004160 tid 2 2024-09-21T09:53:46.538 INFO:tasks.rados.rados.0.smithi033.stdout:807: writing smithi03385152-47 from 7004160 to 7520256 tid 3 2024-09-21T09:53:46.540 INFO:tasks.rados.rados.0.smithi033.stdout:807: writing smithi03385152-47 from 7520256 to 7806976 tid 4 2024-09-21T09:53:46.540 INFO:tasks.rados.rados.0.smithi033.stdout:808: read oid 28 snap -1 2024-09-21T09:53:46.540 INFO:tasks.rados.rados.0.smithi033.stdout:808: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:46.540 INFO:tasks.rados.rados.0.smithi033.stdout:809: snap_remove snap 67 2024-09-21T09:53:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:46 smithi105 ceph-mon[31286]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:53:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:46 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:46 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:46 smithi105 ceph-mon[31286]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:53:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:46 smithi033 ceph-mon[39045]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:53:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:46 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:46 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:46 smithi033 ceph-mon[39045]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:53:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:46 smithi033 ceph-mon[27969]: osdmap e210: 8 total, 8 up, 8 in 2024-09-21T09:53:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:46 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:46 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:46 smithi033 ceph-mon[27969]: osdmap e211: 8 total, 8 up, 8 in 2024-09-21T09:53:47.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:46.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:47.496 INFO:tasks.rados.rados.0.smithi033.stdout:809: done (6 left) 2024-09-21T09:53:47.496 INFO:tasks.rados.rados.0.smithi033.stdout:810: copy_from oid 23 from oid 41 current snap is 69 2024-09-21T09:53:47.497 INFO:tasks.rados.rados.0.smithi033.stdout:805: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:53:47.502 INFO:tasks.rados.rados.0.smithi033.stdout:806: finishing rollback tid 0 to smithi03385152-8 2024-09-21T09:53:47.502 INFO:tasks.rados.rados.0.smithi033.stdout:806: finishing rollback tid 1 to smithi03385152-8 2024-09-21T09:53:47.502 INFO:tasks.rados.rados.0.smithi033.stdout:806: finishing rollback tid 2 to smithi03385152-8 2024-09-21T09:53:47.502 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 215 (ObjNum 255 snap 65 seq_num 255) dirty exists 2024-09-21T09:53:47.503 INFO:tasks.rados.rados.0.smithi033.stdout:803: expect (ObjNum 191 snap 45 seq_num 191) 2024-09-21T09:53:47.610 INFO:tasks.rados.rados.0.smithi033.stdout:804: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:807: finishing write tid 1 to smithi03385152-47 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:807: finishing write tid 2 to smithi03385152-47 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:807: finishing write tid 3 to smithi03385152-47 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:807: finishing write tid 4 to smithi03385152-47 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:807: finishing write tid 5 to smithi03385152-47 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:807: finishing write tid 6 to smithi03385152-47 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 69 (ObjNum 271 snap 69 seq_num 271) dirty exists 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:807: left oid 47 (ObjNum 271 snap 69 seq_num 271) 2024-09-21T09:53:47.910 INFO:tasks.rados.rados.0.smithi033.stdout:808: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[39045]: pgmap v93: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-21T09:53:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[39045]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:53:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:47] ENGINE Bus STARTING 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: CherryPy Checker: 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: The Application mounted at '' has an empty config. 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:47] ENGINE Serving on http://:::9283 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:47] ENGINE Bus STARTED 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[27969]: pgmap v93: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:47.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[27969]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:53:47.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:47 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:47 smithi105 ceph-mon[31286]: pgmap v93: 121 pgs: 121 active+clean; 167 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-21T09:53:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:47 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:47 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:47 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:47 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:47 smithi105 ceph-mon[31286]: osdmap e212: 8 total, 8 up, 8 in 2024-09-21T09:53:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:47 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:48.207 INFO:tasks.rados.rados.0.smithi033.stdout:810: finishing copy_from racing read to smithi03385152-23 2024-09-21T09:53:48.207 INFO:tasks.rados.rados.0.smithi033.stdout:810: finishing copy_from to smithi03385152-23 2024-09-21T09:53:48.207 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 346 (ObjNum 242 snap 63 seq_num 242) dirty exists 2024-09-21T09:53:48.207 INFO:tasks.rados.rados.0.smithi033.stdout:803: done (6 left) 2024-09-21T09:53:48.207 INFO:tasks.rados.rados.0.smithi033.stdout:804: done (5 left) 2024-09-21T09:53:48.207 INFO:tasks.rados.rados.0.smithi033.stdout:805: done (4 left) 2024-09-21T09:53:48.207 INFO:tasks.rados.rados.0.smithi033.stdout:806: done (3 left) 2024-09-21T09:53:48.208 INFO:tasks.rados.rados.0.smithi033.stdout:807: done (2 left) 2024-09-21T09:53:48.208 INFO:tasks.rados.rados.0.smithi033.stdout:808: done (1 left) 2024-09-21T09:53:48.208 INFO:tasks.rados.rados.0.smithi033.stdout:810: done (0 left) 2024-09-21T09:53:48.208 INFO:tasks.rados.rados.0.smithi033.stdout:811: rmattr oid 2 current snap is 69 2024-09-21T09:53:48.208 INFO:tasks.rados.rados.0.smithi033.stdout:811: done (0 left) 2024-09-21T09:53:48.208 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 69 2024-09-21T09:53:48.208 INFO:tasks.rados.rados.0.smithi033.stdout:812: seq_num 272 ranges {0=565248,565248=688128,1253376=729088,1982464=188416} 2024-09-21T09:53:48.211 INFO:tasks.rados.rados.0.smithi033.stdout:812: writing smithi03385152-26 from 0 to 565248 tid 1 2024-09-21T09:53:48.216 INFO:tasks.rados.rados.0.smithi033.stdout:812: writing smithi03385152-26 from 565248 to 1253376 tid 2 2024-09-21T09:53:48.220 INFO:tasks.rados.rados.0.smithi033.stdout:812: writing smithi03385152-26 from 1253376 to 1982464 tid 3 2024-09-21T09:53:48.222 INFO:tasks.rados.rados.0.smithi033.stdout:812: writing smithi03385152-26 from 1982464 to 2170880 tid 4 2024-09-21T09:53:48.222 INFO:tasks.rados.rados.0.smithi033.stdout:813: read oid 20 snap 68 2024-09-21T09:53:48.222 INFO:tasks.rados.rados.0.smithi033.stdout:813: expect deleted 2024-09-21T09:53:48.222 INFO:tasks.rados.rados.0.smithi033.stdout:814: read oid 25 snap -1 2024-09-21T09:53:48.222 INFO:tasks.rados.rados.0.smithi033.stdout:814: expect (ObjNum 243 snap 63 seq_num 243) 2024-09-21T09:53:48.222 INFO:tasks.rados.rados.0.smithi033.stdout:815: rmattr oid 49 current snap is 69 2024-09-21T09:53:48.222 INFO:tasks.rados.rados.0.smithi033.stdout:815: done (3 left) 2024-09-21T09:53:48.222 INFO:tasks.rados.rados.0.smithi033.stdout:816: rmattr oid 30 current snap is 69 2024-09-21T09:53:48.223 INFO:tasks.rados.rados.0.smithi033.stdout:816: done (3 left) 2024-09-21T09:53:48.223 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 69 2024-09-21T09:53:48.223 INFO:tasks.rados.rados.0.smithi033.stdout:812: finishing write tid 1 to smithi03385152-26 2024-09-21T09:53:48.223 INFO:tasks.rados.rados.0.smithi033.stdout:812: finishing write tid 2 to smithi03385152-26 2024-09-21T09:53:48.223 INFO:tasks.rados.rados.0.smithi033.stdout:817: seq_num 273 ranges {1884160=581632,2465792=565248,3031040=532480,3563520=507904} 2024-09-21T09:53:48.235 INFO:tasks.rados.rados.0.smithi033.stdout:817: writing smithi03385152-8 from 1884160 to 2465792 tid 1 2024-09-21T09:53:48.239 INFO:tasks.rados.rados.0.smithi033.stdout:817: writing smithi03385152-8 from 2465792 to 3031040 tid 2 2024-09-21T09:53:48.242 INFO:tasks.rados.rados.0.smithi033.stdout:817: writing smithi03385152-8 from 3031040 to 3563520 tid 3 2024-09-21T09:53:48.245 INFO:tasks.rados.rados.0.smithi033.stdout:817: writing smithi03385152-8 from 3563520 to 4071424 tid 4 2024-09-21T09:53:48.245 INFO:tasks.rados.rados.0.smithi033.stdout:818: rollback oid 18 current snap is 69 2024-09-21T09:53:48.245 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 68 2024-09-21T09:53:48.245 INFO:tasks.rados.rados.0.smithi033.stdout:812: finishing write tid 3 to smithi03385152-26 2024-09-21T09:53:48.245 INFO:tasks.rados.rados.0.smithi033.stdout:812: finishing write tid 4 to smithi03385152-26 2024-09-21T09:53:48.245 INFO:tasks.rados.rados.0.smithi033.stdout:812: finishing write tid 5 to smithi03385152-26 2024-09-21T09:53:48.246 INFO:tasks.rados.rados.0.smithi033.stdout:812: finishing write tid 6 to smithi03385152-26 2024-09-21T09:53:48.246 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 361 (ObjNum 272 snap 69 seq_num 272) dirty exists 2024-09-21T09:53:48.246 INFO:tasks.rados.rados.0.smithi033.stdout:812: left oid 26 (ObjNum 272 snap 69 seq_num 272) 2024-09-21T09:53:48.246 INFO:tasks.rados.rados.0.smithi033.stdout:814: expect (ObjNum 243 snap 63 seq_num 243) 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:817: finishing write tid 1 to smithi03385152-8 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:817: finishing write tid 2 to smithi03385152-8 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:817: finishing write tid 3 to smithi03385152-8 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:817: finishing write tid 4 to smithi03385152-8 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:817: finishing write tid 5 to smithi03385152-8 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:818: finishing rollback tid 0 to smithi03385152-18 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:818: finishing rollback tid 1 to smithi03385152-18 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:818: finishing rollback tid 2 to smithi03385152-18 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 122 (ObjNum 138 snap 32 seq_num 138) dirty exists 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:817: finishing write tid 6 to smithi03385152-8 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 224 (ObjNum 273 snap 69 seq_num 273) dirty exists 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:817: left oid 8 (ObjNum 273 snap 69 seq_num 273) 2024-09-21T09:53:48.443 INFO:tasks.rados.rados.0.smithi033.stdout:812: done (4 left) 2024-09-21T09:53:48.444 INFO:tasks.rados.rados.0.smithi033.stdout:813: done (3 left) 2024-09-21T09:53:48.444 INFO:tasks.rados.rados.0.smithi033.stdout:814: done (2 left) 2024-09-21T09:53:48.444 INFO:tasks.rados.rados.0.smithi033.stdout:817: done (1 left) 2024-09-21T09:53:48.444 INFO:tasks.rados.rados.0.smithi033.stdout:818: done (0 left) 2024-09-21T09:53:48.444 INFO:tasks.rados.rados.0.smithi033.stdout:819: delete oid 33 current snap is 69 2024-09-21T09:53:48.444 INFO:tasks.rados.rados.0.smithi033.stdout:819: done (0 left) 2024-09-21T09:53:48.444 INFO:tasks.rados.rados.0.smithi033.stdout:820: rmattr oid 3 current snap is 69 2024-09-21T09:53:48.444 INFO:tasks.rados.rados.0.smithi033.stdout:820: done (0 left) 2024-09-21T09:53:48.445 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 69 2024-09-21T09:53:48.445 INFO:tasks.rados.rados.0.smithi033.stdout:821: seq_num 275 ranges {0=614400,614400=786432,1400832=499712,1900544=319488} 2024-09-21T09:53:48.447 INFO:tasks.rados.rados.0.smithi033.stdout:821: writing smithi03385152-44 from 0 to 614400 tid 1 2024-09-21T09:53:48.452 INFO:tasks.rados.rados.0.smithi033.stdout:821: writing smithi03385152-44 from 614400 to 1400832 tid 2 2024-09-21T09:53:48.455 INFO:tasks.rados.rados.0.smithi033.stdout:821: writing smithi03385152-44 from 1400832 to 1900544 tid 3 2024-09-21T09:53:48.457 INFO:tasks.rados.rados.0.smithi033.stdout:821: writing smithi03385152-44 from 1900544 to 2220032 tid 4 2024-09-21T09:53:48.457 INFO:tasks.rados.rados.0.smithi033.stdout:822: rollback oid 13 current snap is 69 2024-09-21T09:53:48.457 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 13 to 68 2024-09-21T09:53:48.457 INFO:tasks.rados.rados.0.smithi033.stdout:821: finishing write tid 1 to smithi03385152-44 2024-09-21T09:53:48.457 INFO:tasks.rados.rados.0.smithi033.stdout:821: finishing write tid 2 to smithi03385152-44 2024-09-21T09:53:48.457 INFO:tasks.rados.rados.0.smithi033.stdout:823: read oid 37 snap -1 2024-09-21T09:53:48.458 INFO:tasks.rados.rados.0.smithi033.stdout:823: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:48.458 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 69 2024-09-21T09:53:48.458 INFO:tasks.rados.rados.0.smithi033.stdout:824: seq_num 276 ranges {6955008=630784,7585792=671744,8257536=704512,8962048=229376} 2024-09-21T09:53:48.499 INFO:tasks.rados.rados.0.smithi033.stdout:824: writing smithi03385152-14 from 6955008 to 7585792 tid 1 2024-09-21T09:53:48.503 INFO:tasks.rados.rados.0.smithi033.stdout:824: writing smithi03385152-14 from 7585792 to 8257536 tid 2 2024-09-21T09:53:48.507 INFO:tasks.rados.rados.0.smithi033.stdout:824: writing smithi03385152-14 from 8257536 to 8962048 tid 3 2024-09-21T09:53:48.508 INFO:tasks.rados.rados.0.smithi033.stdout:824: writing smithi03385152-14 from 8962048 to 9191424 tid 4 2024-09-21T09:53:48.508 INFO:tasks.rados.rados.0.smithi033.stdout:825: setattr oid 4 current snap is 69 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:821: finishing write tid 3 to smithi03385152-44 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:821: finishing write tid 4 to smithi03385152-44 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:821: finishing write tid 5 to smithi03385152-44 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:822: finishing rollback tid 0 to smithi03385152-13 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:822: finishing rollback tid 1 to smithi03385152-13 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:822: finishing rollback tid 2 to smithi03385152-13 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 588 (ObjNum 248 snap 65 seq_num 248) dirty exists 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:821: finishing write tid 6 to smithi03385152-44 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 151 (ObjNum 275 snap 69 seq_num 275) dirty exists 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:821: left oid 44 (ObjNum 275 snap 69 seq_num 275) 2024-09-21T09:53:48.509 INFO:tasks.rados.rados.0.smithi033.stdout:823: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[39045]: Standby manager daemon y restarted 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[39045]: Standby manager daemon y started 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[27969]: Standby manager daemon y restarted 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[27969]: Standby manager daemon y started 2024-09-21T09:53:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:48.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:48.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:48.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:48 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:48.809 INFO:tasks.rados.rados.0.smithi033.stdout:824: finishing write tid 1 to smithi03385152-14 2024-09-21T09:53:48.809 INFO:tasks.rados.rados.0.smithi033.stdout:824: finishing write tid 2 to smithi03385152-14 2024-09-21T09:53:48.809 INFO:tasks.rados.rados.0.smithi033.stdout:824: finishing write tid 3 to smithi03385152-14 2024-09-21T09:53:48.809 INFO:tasks.rados.rados.0.smithi033.stdout:824: finishing write tid 4 to smithi03385152-14 2024-09-21T09:53:48.809 INFO:tasks.rados.rados.0.smithi033.stdout:824: finishing write tid 5 to smithi03385152-14 2024-09-21T09:53:48.809 INFO:tasks.rados.rados.0.smithi033.stdout:824: finishing write tid 6 to smithi03385152-14 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 561 (ObjNum 276 snap 69 seq_num 276) dirty exists 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:824: left oid 14 (ObjNum 276 snap 69 seq_num 276) 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:821: done (4 left) 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:822: done (3 left) 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:823: done (2 left) 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:824: done (1 left) 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:826: copy_from oid 13 from oid 46 current snap is 69 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:827: read oid 22 snap 68 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:827: expect deleted 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:828: rmattr oid 20 current snap is 69 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:828: done (3 left) 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 69 2024-09-21T09:53:48.810 INFO:tasks.rados.rados.0.smithi033.stdout:829: seq_num 277 ranges {0=647168,647168=557056,1204224=516096,1720320=532480} 2024-09-21T09:53:48.812 INFO:tasks.rados.rados.0.smithi033.stdout:829: writing smithi03385152-3 from 0 to 647168 tid 1 2024-09-21T09:53:48.815 INFO:tasks.rados.rados.0.smithi033.stdout:829: writing smithi03385152-3 from 647168 to 1204224 tid 2 2024-09-21T09:53:48.818 INFO:tasks.rados.rados.0.smithi033.stdout:829: writing smithi03385152-3 from 1204224 to 1720320 tid 3 2024-09-21T09:53:48.821 INFO:tasks.rados.rados.0.smithi033.stdout:829: writing smithi03385152-3 from 1720320 to 2252800 tid 4 2024-09-21T09:53:48.821 INFO:tasks.rados.rados.0.smithi033.stdout:830: rmattr oid 17 current snap is 69 2024-09-21T09:53:48.821 INFO:tasks.rados.rados.0.smithi033.stdout:830: done (4 left) 2024-09-21T09:53:48.821 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 69 2024-09-21T09:53:48.821 INFO:tasks.rados.rados.0.smithi033.stdout:831: seq_num 278 ranges {1851392=663552,2514944=442368,2957312=720896,3678208=442368} 2024-09-21T09:53:48.835 INFO:tasks.rados.rados.0.smithi033.stdout:831: writing smithi03385152-17 from 1851392 to 2514944 tid 1 2024-09-21T09:53:48.837 INFO:tasks.rados.rados.0.smithi033.stdout:831: writing smithi03385152-17 from 2514944 to 2957312 tid 2 2024-09-21T09:53:48.841 INFO:tasks.rados.rados.0.smithi033.stdout:831: writing smithi03385152-17 from 2957312 to 3678208 tid 3 2024-09-21T09:53:48.844 INFO:tasks.rados.rados.0.smithi033.stdout:831: writing smithi03385152-17 from 3678208 to 4120576 tid 4 2024-09-21T09:53:48.844 INFO:tasks.rados.rados.0.smithi033.stdout:832: read oid 10 snap 68 2024-09-21T09:53:48.844 INFO:tasks.rados.rados.0.smithi033.stdout:832: expect deleted 2024-09-21T09:53:48.844 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 69 2024-09-21T09:53:48.844 INFO:tasks.rados.rados.0.smithi033.stdout:833: seq_num 279 ranges {3219456=679936,3899392=720896,4620288=483328,5103616=401408} 2024-09-21T09:53:48.865 INFO:tasks.rados.rados.0.smithi033.stdout:833: writing smithi03385152-35 from 3219456 to 3899392 tid 1 2024-09-21T09:53:48.869 INFO:tasks.rados.rados.0.smithi033.stdout:833: writing smithi03385152-35 from 3899392 to 4620288 tid 2 2024-09-21T09:53:48.872 INFO:tasks.rados.rados.0.smithi033.stdout:833: writing smithi03385152-35 from 4620288 to 5103616 tid 3 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:833: writing smithi03385152-35 from 5103616 to 5505024 tid 4 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:834: read oid 1 snap 68 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:834: expect deleted 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:835: setattr oid 43 current snap is 69 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 153 (ObjNum 260 snap 66 seq_num 260) dirty exists 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:826: finishing copy_from racing read to smithi03385152-13 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:829: finishing write tid 1 to smithi03385152-3 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:829: finishing write tid 2 to smithi03385152-3 2024-09-21T09:53:48.875 INFO:tasks.rados.rados.0.smithi033.stdout:826: finishing copy_from to smithi03385152-13 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 347 (ObjNum 253 snap 65 seq_num 253) dirty exists 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:829: finishing write tid 3 to smithi03385152-3 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:829: finishing write tid 4 to smithi03385152-3 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:829: finishing write tid 5 to smithi03385152-3 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:829: finishing write tid 6 to smithi03385152-3 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 566 (ObjNum 277 snap 69 seq_num 277) dirty exists 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:829: left oid 3 (ObjNum 277 snap 69 seq_num 277) 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:831: finishing write tid 1 to smithi03385152-17 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:831: finishing write tid 2 to smithi03385152-17 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:831: finishing write tid 3 to smithi03385152-17 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:831: finishing write tid 4 to smithi03385152-17 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:831: finishing write tid 5 to smithi03385152-17 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:831: finishing write tid 6 to smithi03385152-17 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 366 (ObjNum 278 snap 69 seq_num 278) dirty exists 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:831: left oid 17 (ObjNum 278 snap 69 seq_num 278) 2024-09-21T09:53:48.876 INFO:tasks.rados.rados.0.smithi033.stdout:833: finishing write tid 1 to smithi03385152-35 2024-09-21T09:53:48.877 INFO:tasks.rados.rados.0.smithi033.stdout:833: finishing write tid 2 to smithi03385152-35 2024-09-21T09:53:48.877 INFO:tasks.rados.rados.0.smithi033.stdout:833: finishing write tid 3 to smithi03385152-35 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:833: finishing write tid 4 to smithi03385152-35 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:833: finishing write tid 5 to smithi03385152-35 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:825: done (8 left) 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:826: done (7 left) 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:827: done (6 left) 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:829: done (5 left) 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:831: done (4 left) 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:832: done (3 left) 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:834: done (2 left) 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:836: read oid 12 snap 68 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:836: expect (ObjNum 262 snap 67 seq_num 262) 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:837: rmattr oid 34 current snap is 69 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:838: rmattr oid 8 current snap is 69 2024-09-21T09:53:48.878 INFO:tasks.rados.rados.0.smithi033.stdout:838: done (4 left) 2024-09-21T09:53:48.879 INFO:tasks.rados.rados.0.smithi033.stdout:839: rollback oid 50 current snap is 69 2024-09-21T09:53:48.879 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 50 to 68 2024-09-21T09:53:48.879 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 69 2024-09-21T09:53:48.879 INFO:tasks.rados.rados.0.smithi033.stdout:840: seq_num 280 ranges {0=696320,696320=606208,1302528=688128,1990656=311296} 2024-09-21T09:53:48.882 INFO:tasks.rados.rados.0.smithi033.stdout:840: writing smithi03385152-9 from 0 to 696320 tid 1 2024-09-21T09:53:48.886 INFO:tasks.rados.rados.0.smithi033.stdout:840: writing smithi03385152-9 from 696320 to 1302528 tid 2 2024-09-21T09:53:48.890 INFO:tasks.rados.rados.0.smithi033.stdout:840: writing smithi03385152-9 from 1302528 to 1990656 tid 3 2024-09-21T09:53:48.892 INFO:tasks.rados.rados.0.smithi033.stdout:840: writing smithi03385152-9 from 1990656 to 2301952 tid 4 2024-09-21T09:53:48.892 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 69 2024-09-21T09:53:48.892 INFO:tasks.rados.rados.0.smithi033.stdout:833: finishing write tid 6 to smithi03385152-35 2024-09-21T09:53:48.892 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 595 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:53:48.892 INFO:tasks.rados.rados.0.smithi033.stdout:833: left oid 35 (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:53:48.893 INFO:tasks.rados.rados.0.smithi033.stdout:839: finishing rollback tid 0 to smithi03385152-50 2024-09-21T09:53:48.893 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 568 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-21T09:53:48.893 INFO:tasks.rados.rados.0.smithi033.stdout:839: finishing rollback tid 1 to smithi03385152-50 2024-09-21T09:53:48.893 INFO:tasks.rados.rados.0.smithi033.stdout:839: finishing rollback tid 2 to smithi03385152-50 2024-09-21T09:53:48.893 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 170 (ObjNum 224 snap 52 seq_num 224) dirty exists 2024-09-21T09:53:48.893 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 360 (ObjNum 266 snap 67 seq_num 266) dirty exists 2024-09-21T09:53:48.893 INFO:tasks.rados.rados.0.smithi033.stdout:840: finishing write tid 1 to smithi03385152-9 2024-09-21T09:53:48.893 INFO:tasks.rados.rados.0.smithi033.stdout:841: seq_num 281 ranges {3710976=712704,4423680=491520,4915200=491520,5406720=622592} 2024-09-21T09:53:48.916 INFO:tasks.rados.rados.0.smithi033.stdout:841: writing smithi03385152-4 from 3710976 to 4423680 tid 1 2024-09-21T09:53:48.919 INFO:tasks.rados.rados.0.smithi033.stdout:841: writing smithi03385152-4 from 4423680 to 4915200 tid 2 2024-09-21T09:53:48.922 INFO:tasks.rados.rados.0.smithi033.stdout:841: writing smithi03385152-4 from 4915200 to 5406720 tid 3 2024-09-21T09:53:48.925 INFO:tasks.rados.rados.0.smithi033.stdout:841: writing smithi03385152-4 from 5406720 to 6029312 tid 4 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:833: done (6 left) 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:835: done (5 left) 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:837: done (4 left) 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:839: done (3 left) 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:842: setattr oid 32 current snap is 69 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:840: finishing write tid 2 to smithi03385152-9 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:840: finishing write tid 3 to smithi03385152-9 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:840: finishing write tid 4 to smithi03385152-9 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:840: finishing write tid 5 to smithi03385152-9 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:840: finishing write tid 6 to smithi03385152-9 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 351 (ObjNum 280 snap 69 seq_num 280) dirty exists 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:840: left oid 9 (ObjNum 280 snap 69 seq_num 280) 2024-09-21T09:53:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:836: expect (ObjNum 262 snap 67 seq_num 262) 2024-09-21T09:53:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:48 smithi105 ceph-mon[31286]: Standby manager daemon y restarted 2024-09-21T09:53:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:48 smithi105 ceph-mon[31286]: Standby manager daemon y started 2024-09-21T09:53:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:48 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-21T09:53:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:48 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:48 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-21T09:53:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:48 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:49.264 INFO:tasks.rados.rados.0.smithi033.stdout:841: finishing write tid 1 to smithi03385152-4 2024-09-21T09:53:49.264 INFO:tasks.rados.rados.0.smithi033.stdout:841: finishing write tid 2 to smithi03385152-4 2024-09-21T09:53:49.264 INFO:tasks.rados.rados.0.smithi033.stdout:841: finishing write tid 3 to smithi03385152-4 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:841: finishing write tid 4 to smithi03385152-4 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:841: finishing write tid 5 to smithi03385152-4 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:841: finishing write tid 6 to smithi03385152-4 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 158 (ObjNum 281 snap 69 seq_num 281) dirty exists 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:841: left oid 4 (ObjNum 281 snap 69 seq_num 281) 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:836: done (3 left) 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:840: done (2 left) 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:841: done (1 left) 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:843: read oid 28 snap -1 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:843: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:844: copy_from oid 9 from oid 7 current snap is 69 2024-09-21T09:53:49.265 INFO:tasks.rados.rados.0.smithi033.stdout:845: delete oid 31 current snap is 69 2024-09-21T09:53:49.267 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 96 (ObjNum 199 snap 47 seq_num 199) dirty exists 2024-09-21T09:53:49.268 INFO:tasks.rados.rados.0.smithi033.stdout:842: done (3 left) 2024-09-21T09:53:49.268 INFO:tasks.rados.rados.0.smithi033.stdout:845: done (2 left) 2024-09-21T09:53:49.268 INFO:tasks.rados.rados.0.smithi033.stdout:846: rollback oid 44 current snap is 69 2024-09-21T09:53:49.268 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 44 to 68 2024-09-21T09:53:49.268 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 69 2024-09-21T09:53:49.269 INFO:tasks.rados.rados.0.smithi033.stdout:847: seq_num 283 ranges {2187264=745472,2932736=712704,3645440=458752,4104192=434176} 2024-09-21T09:53:49.285 INFO:tasks.rados.rados.0.smithi033.stdout:847: writing smithi03385152-50 from 2187264 to 2932736 tid 1 2024-09-21T09:53:49.290 INFO:tasks.rados.rados.0.smithi033.stdout:847: writing smithi03385152-50 from 2932736 to 3645440 tid 2 2024-09-21T09:53:49.293 INFO:tasks.rados.rados.0.smithi033.stdout:847: writing smithi03385152-50 from 3645440 to 4104192 tid 3 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:847: writing smithi03385152-50 from 4104192 to 4538368 tid 4 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:848: copy_from oid 41 from oid 35 current snap is 69 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:846: finishing rollback tid 0 to smithi03385152-44 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:849: rollback oid 2 current snap is 69 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 68 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:846: finishing rollback tid 1 to smithi03385152-44 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 160 (ObjNum 129 snap 0 seq_num 90941488) dirty dne 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:844: finishing copy_from racing read to smithi03385152-9 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:846: done (5 left) 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 69 2024-09-21T09:53:49.297 INFO:tasks.rados.rados.0.smithi033.stdout:850: seq_num 284 ranges {1302528=761856,2064384=598016,2662400=671744,3334144=335872} 2024-09-21T09:53:49.308 INFO:tasks.rados.rados.0.smithi033.stdout:850: writing smithi03385152-42 from 1302528 to 2064384 tid 1 2024-09-21T09:53:49.311 INFO:tasks.rados.rados.0.smithi033.stdout:850: writing smithi03385152-42 from 2064384 to 2662400 tid 2 2024-09-21T09:53:49.315 INFO:tasks.rados.rados.0.smithi033.stdout:850: writing smithi03385152-42 from 2662400 to 3334144 tid 3 2024-09-21T09:53:49.317 INFO:tasks.rados.rados.0.smithi033.stdout:850: writing smithi03385152-42 from 3334144 to 3670016 tid 4 2024-09-21T09:53:49.318 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 69 2024-09-21T09:53:49.318 INFO:tasks.rados.rados.0.smithi033.stdout:843: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:49.617 INFO:tasks.rados.rados.0.smithi033.stdout:844: finishing copy_from to smithi03385152-9 2024-09-21T09:53:49.617 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 494 (ObjNum 254 snap 65 seq_num 254) dirty exists 2024-09-21T09:53:49.617 INFO:tasks.rados.rados.0.smithi033.stdout:847: finishing write tid 1 to smithi03385152-50 2024-09-21T09:53:49.617 INFO:tasks.rados.rados.0.smithi033.stdout:847: finishing write tid 2 to smithi03385152-50 2024-09-21T09:53:49.618 INFO:tasks.rados.rados.0.smithi033.stdout:847: finishing write tid 3 to smithi03385152-50 2024-09-21T09:53:49.618 INFO:tasks.rados.rados.0.smithi033.stdout:851: seq_num 285 ranges {6774784=778240,7553024=475136,8028160=475136,8503296=655360} 2024-09-21T09:53:49.658 INFO:tasks.rados.rados.0.smithi033.stdout:851: writing smithi03385152-24 from 6774784 to 7553024 tid 1 2024-09-21T09:53:49.661 INFO:tasks.rados.rados.0.smithi033.stdout:851: writing smithi03385152-24 from 7553024 to 8028160 tid 2 2024-09-21T09:53:49.664 INFO:tasks.rados.rados.0.smithi033.stdout:851: writing smithi03385152-24 from 8028160 to 8503296 tid 3 2024-09-21T09:53:49.668 INFO:tasks.rados.rados.0.smithi033.stdout:851: writing smithi03385152-24 from 8503296 to 9158656 tid 4 2024-09-21T09:53:49.668 INFO:tasks.rados.rados.0.smithi033.stdout:843: done (6 left) 2024-09-21T09:53:49.668 INFO:tasks.rados.rados.0.smithi033.stdout:844: done (5 left) 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:852: snap_create 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:847: finishing write tid 4 to smithi03385152-50 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:847: finishing write tid 5 to smithi03385152-50 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:849: finishing rollback tid 0 to smithi03385152-2 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:849: finishing rollback tid 1 to smithi03385152-2 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:849: finishing rollback tid 2 to smithi03385152-2 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 179 (ObjNum 228 snap 53 seq_num 228) dirty exists 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:847: finishing write tid 6 to smithi03385152-50 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 175 (ObjNum 283 snap 69 seq_num 283) dirty exists 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:847: left oid 50 (ObjNum 283 snap 69 seq_num 283) 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:850: finishing write tid 1 to smithi03385152-42 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:848: finishing copy_from racing read to smithi03385152-41 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:850: finishing write tid 2 to smithi03385152-42 2024-09-21T09:53:49.669 INFO:tasks.rados.rados.0.smithi033.stdout:848: finishing copy_from to smithi03385152-41 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 595 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:850: finishing write tid 3 to smithi03385152-42 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:850: finishing write tid 4 to smithi03385152-42 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:850: finishing write tid 5 to smithi03385152-42 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:850: finishing write tid 6 to smithi03385152-42 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 598 (ObjNum 284 snap 69 seq_num 284) dirty exists 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:850: left oid 42 (ObjNum 284 snap 69 seq_num 284) 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:851: finishing write tid 1 to smithi03385152-24 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:851: finishing write tid 2 to smithi03385152-24 2024-09-21T09:53:49.670 INFO:tasks.rados.rados.0.smithi033.stdout:851: finishing write tid 3 to smithi03385152-24 2024-09-21T09:53:49.671 INFO:tasks.rados.rados.0.smithi033.stdout:851: finishing write tid 4 to smithi03385152-24 2024-09-21T09:53:49.671 INFO:tasks.rados.rados.0.smithi033.stdout:851: finishing write tid 5 to smithi03385152-24 2024-09-21T09:53:49.671 INFO:tasks.rados.rados.0.smithi033.stdout:851: finishing write tid 6 to smithi03385152-24 2024-09-21T09:53:49.671 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 157 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:53:49.671 INFO:tasks.rados.rados.0.smithi033.stdout:851: left oid 24 (ObjNum 285 snap 69 seq_num 285) 2024-09-21T09:53:49.680 INFO:tasks.rados.rados.0.smithi033.stdout:847: done (5 left) 2024-09-21T09:53:49.681 INFO:tasks.rados.rados.0.smithi033.stdout:848: done (4 left) 2024-09-21T09:53:49.681 INFO:tasks.rados.rados.0.smithi033.stdout:849: done (3 left) 2024-09-21T09:53:49.681 INFO:tasks.rados.rados.0.smithi033.stdout:850: done (2 left) 2024-09-21T09:53:49.681 INFO:tasks.rados.rados.0.smithi033.stdout:851: done (1 left) 2024-09-21T09:53:49.681 INFO:tasks.rados.rados.0.smithi033.stdout:852: done (0 left) 2024-09-21T09:53:49.681 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 70 2024-09-21T09:53:49.681 INFO:tasks.rados.rados.0.smithi033.stdout:853: seq_num 286 ranges {1794048=794624,2588672=761856,3350528=679936,4030464=163840} 2024-09-21T09:53:49.694 INFO:tasks.rados.rados.0.smithi033.stdout:853: writing smithi03385152-45 from 1794048 to 2588672 tid 1 2024-09-21T09:53:49.699 INFO:tasks.rados.rados.0.smithi033.stdout:853: writing smithi03385152-45 from 2588672 to 3350528 tid 2 2024-09-21T09:53:49.703 INFO:tasks.rados.rados.0.smithi033.stdout:853: writing smithi03385152-45 from 3350528 to 4030464 tid 3 2024-09-21T09:53:49.704 INFO:tasks.rados.rados.0.smithi033.stdout:853: writing smithi03385152-45 from 4030464 to 4194304 tid 4 2024-09-21T09:53:49.704 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 70 2024-09-21T09:53:49.704 INFO:tasks.rados.rados.0.smithi033.stdout:854: seq_num 287 ranges {0=8192} 2024-09-21T09:53:49.704 INFO:tasks.rados.rados.0.smithi033.stdout:854: writing smithi03385152-10 from 0 to 8192 tid 1 2024-09-21T09:53:49.704 INFO:tasks.rados.rados.0.smithi033.stdout:855: delete oid 7 current snap is 70 2024-09-21T09:53:49.704 INFO:tasks.rados.rados.0.smithi033.stdout:853: finishing write tid 1 to smithi03385152-45 2024-09-21T09:53:49.704 INFO:tasks.rados.rados.0.smithi033.stdout:853: finishing write tid 2 to smithi03385152-45 2024-09-21T09:53:49.705 INFO:tasks.rados.rados.0.smithi033.stdout:854: finishing write tid 1 to smithi03385152-10 2024-09-21T09:53:49.705 INFO:tasks.rados.rados.0.smithi033.stdout:854: finishing write tid 2 to smithi03385152-10 2024-09-21T09:53:49.707 INFO:tasks.rados.rados.0.smithi033.stdout:853: finishing write tid 3 to smithi03385152-45 2024-09-21T09:53:49.707 INFO:tasks.rados.rados.0.smithi033.stdout:854: finishing write tid 3 to smithi03385152-10 2024-09-21T09:53:49.707 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 159 (ObjNum 287 snap 70 seq_num 287) dirty exists 2024-09-21T09:53:49.707 INFO:tasks.rados.rados.0.smithi033.stdout:854: left oid 10 (ObjNum 287 snap 70 seq_num 287) 2024-09-21T09:53:49.707 INFO:tasks.rados.rados.0.smithi033.stdout:853: finishing write tid 4 to smithi03385152-45 2024-09-21T09:53:49.707 INFO:tasks.rados.rados.0.smithi033.stdout:853: finishing write tid 5 to smithi03385152-45 2024-09-21T09:53:49.708 INFO:tasks.rados.rados.0.smithi033.stdout:853: finishing write tid 6 to smithi03385152-45 2024-09-21T09:53:49.708 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 574 (ObjNum 286 snap 70 seq_num 286) dirty exists 2024-09-21T09:53:49.708 INFO:tasks.rados.rados.0.smithi033.stdout:853: left oid 45 (ObjNum 286 snap 70 seq_num 286) 2024-09-21T09:53:49.711 INFO:tasks.rados.rados.0.smithi033.stdout:853: done (2 left) 2024-09-21T09:53:49.711 INFO:tasks.rados.rados.0.smithi033.stdout:854: done (1 left) 2024-09-21T09:53:49.711 INFO:tasks.rados.rados.0.smithi033.stdout:855: done (0 left) 2024-09-21T09:53:49.711 INFO:tasks.rados.rados.0.smithi033.stdout:856: rmattr oid 17 current snap is 70 2024-09-21T09:53:49.711 INFO:tasks.rados.rados.0.smithi033.stdout:856: done (0 left) 2024-09-21T09:53:49.711 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 70 2024-09-21T09:53:49.711 INFO:tasks.rados.rados.0.smithi033.stdout:857: seq_num 289 ranges {3522560=40960} 2024-09-21T09:53:49.731 INFO:tasks.rados.rados.0.smithi033.stdout:857: writing smithi03385152-30 from 3522560 to 3563520 tid 1 2024-09-21T09:53:49.731 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 70 2024-09-21T09:53:49.731 INFO:tasks.rados.rados.0.smithi033.stdout:858: seq_num 290 ranges {2523136=57344} 2024-09-21T09:53:49.745 INFO:tasks.rados.rados.0.smithi033.stdout:858: writing smithi03385152-27 from 2523136 to 2580480 tid 1 2024-09-21T09:53:49.745 INFO:tasks.rados.rados.0.smithi033.stdout:859: delete oid 20 current snap is 70 2024-09-21T09:53:49.745 INFO:tasks.rados.rados.0.smithi033.stdout:857: finishing write tid 1 to smithi03385152-30 2024-09-21T09:53:49.745 INFO:tasks.rados.rados.0.smithi033.stdout:857: finishing write tid 2 to smithi03385152-30 2024-09-21T09:53:49.745 INFO:tasks.rados.rados.0.smithi033.stdout:857: finishing write tid 3 to smithi03385152-30 2024-09-21T09:53:49.745 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 602 (ObjNum 289 snap 70 seq_num 289) dirty exists 2024-09-21T09:53:49.746 INFO:tasks.rados.rados.0.smithi033.stdout:857: left oid 30 (ObjNum 289 snap 70 seq_num 289) 2024-09-21T09:53:49.746 INFO:tasks.rados.rados.0.smithi033.stdout:857: done (2 left) 2024-09-21T09:53:49.746 INFO:tasks.rados.rados.0.smithi033.stdout:859: done (1 left) 2024-09-21T09:53:49.746 INFO:tasks.rados.rados.0.smithi033.stdout:860: snap_create 2024-09-21T09:53:49.747 INFO:tasks.rados.rados.0.smithi033.stdout:858: finishing write tid 1 to smithi03385152-27 2024-09-21T09:53:49.747 INFO:tasks.rados.rados.0.smithi033.stdout:858: finishing write tid 2 to smithi03385152-27 2024-09-21T09:53:49.748 INFO:tasks.rados.rados.0.smithi033.stdout:858: finishing write tid 3 to smithi03385152-27 2024-09-21T09:53:49.748 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 309 (ObjNum 290 snap 70 seq_num 290) dirty exists 2024-09-21T09:53:49.748 INFO:tasks.rados.rados.0.smithi033.stdout:858: left oid 27 (ObjNum 290 snap 70 seq_num 290) 2024-09-21T09:53:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:49 smithi033 ceph-mon[39045]: pgmap v96: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.9 MiB/s wr, 9 op/s 2024-09-21T09:53:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:49 smithi033 ceph-mon[39045]: mgrmap e29: x(active, since 62s), standbys: y 2024-09-21T09:53:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:49 smithi033 ceph-mon[27969]: pgmap v96: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.9 MiB/s wr, 9 op/s 2024-09-21T09:53:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:49 smithi033 ceph-mon[27969]: mgrmap e29: x(active, since 62s), standbys: y 2024-09-21T09:53:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:49 smithi105 ceph-mon[31286]: pgmap v96: 121 pgs: 121 active+clean; 128 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.9 MiB/s wr, 9 op/s 2024-09-21T09:53:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:49 smithi105 ceph-mon[31286]: mgrmap e29: x(active, since 62s), standbys: y 2024-09-21T09:53:50.180 INFO:tasks.rados.rados.0.smithi033.stdout:858: done (1 left) 2024-09-21T09:53:50.180 INFO:tasks.rados.rados.0.smithi033.stdout:860: done (0 left) 2024-09-21T09:53:50.181 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 71 2024-09-21T09:53:50.181 INFO:tasks.rados.rados.0.smithi033.stdout:861: seq_num 292 ranges {1851392=98304} 2024-09-21T09:53:50.191 INFO:tasks.rados.rados.0.smithi033.stdout:861: writing smithi03385152-13 from 1851392 to 1949696 tid 1 2024-09-21T09:53:50.191 INFO:tasks.rados.rados.0.smithi033.stdout:862: delete oid 50 current snap is 71 2024-09-21T09:53:50.193 INFO:tasks.rados.rados.0.smithi033.stdout:862: done (1 left) 2024-09-21T09:53:50.193 INFO:tasks.rados.rados.0.smithi033.stdout:863: rollback oid 21 current snap is 71 2024-09-21T09:53:50.193 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 68 2024-09-21T09:53:50.194 INFO:tasks.rados.rados.0.smithi033.stdout:864: snap_remove snap 69 2024-09-21T09:53:50.361 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:50] ENGINE Bus STOPPING 2024-09-21T09:53:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:50.530 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:53:50.180+0000 7f7cfd744640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-21T09:53:50.530 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ignoring --setuser ceph since I am not root 2024-09-21T09:53:50.530 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ignoring --setgroup ceph since I am not root 2024-09-21T09:53:50.531 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mgr[67143]: -- 172.21.15.105:0/2649523211 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5619a34bb0e0 con 0x5619a3498800 2024-09-21T09:53:50.678 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:53:50.678 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:50] ENGINE Bus STOPPED 2024-09-21T09:53:50.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:53:50.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:50.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:50.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:50.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:50.949 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:50.949 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:50.949 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:50.949 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:50.949 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: Activating manager daemon y 2024-09-21T09:53:50.949 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:53:50.949 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.24823 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:53:50.950 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: mgrmap e30: y(active, starting, since 0.00711193s) 2024-09-21T09:53:50.950 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:50.950 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:50.950 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:50.950 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:53:50.950 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:53:50.950 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:50.951 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:50.951 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:50.951 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:50.951 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:50.951 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:50.951 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:50.952 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:50.952 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:50.952 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:50.952 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:50.952 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:50.952 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[39045]: Manager daemon y is now available 2024-09-21T09:53:50.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:53:50.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:50.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:50.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:50.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:50.953 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:50.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:50.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:50.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:50.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: Activating manager daemon y 2024-09-21T09:53:50.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:53:50.954 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.24823 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:53:50.955 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: mgrmap e30: y(active, starting, since 0.00711193s) 2024-09-21T09:53:50.955 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:50.955 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:50.955 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:50.955 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:53:50.955 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:53:50.955 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:50.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:50.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:50.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:50.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:50.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:50.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:50.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:50.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:50.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:50.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:50.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:50.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-mon[27969]: Manager daemon y is now available 2024-09-21T09:53:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: osdmap e213: 8 total, 8 up, 8 in 2024-09-21T09:53:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:53:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:53:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' 2024-09-21T09:53:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 172.21.15.105:0/1828076828' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-21T09:53:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: Activating manager daemon y 2024-09-21T09:53:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: osdmap e214: 8 total, 8 up, 8 in 2024-09-21T09:53:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.24823 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-21T09:53:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: mgrmap e30: y(active, starting, since 0.00711193s) 2024-09-21T09:53:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:53:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:53:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:53:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:53:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:53:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:53:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:53:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:53:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:53:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:53:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:53:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:53:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:53:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:53:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:53:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:53:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:53:51.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:50 smithi105 ceph-mon[31286]: Manager daemon y is now available 2024-09-21T09:53:51.195 INFO:tasks.rados.rados.0.smithi033.stdout:864: done (2 left) 2024-09-21T09:53:51.195 INFO:tasks.rados.rados.0.smithi033.stdout:865: snap_create 2024-09-21T09:53:51.196 INFO:tasks.rados.rados.0.smithi033.stdout:861: finishing write tid 1 to smithi03385152-13 2024-09-21T09:53:51.196 INFO:tasks.rados.rados.0.smithi033.stdout:861: finishing write tid 2 to smithi03385152-13 2024-09-21T09:53:51.196 INFO:tasks.rados.rados.0.smithi033.stdout:863: finishing rollback tid 0 to smithi03385152-21 2024-09-21T09:53:51.196 INFO:tasks.rados.rados.0.smithi033.stdout:863: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:53:51.196 INFO:tasks.rados.rados.0.smithi033.stdout:863: finishing rollback tid 2 to smithi03385152-21 2024-09-21T09:53:51.197 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 349 (ObjNum 112 snap 25 seq_num 112) dirty exists 2024-09-21T09:53:51.197 INFO:tasks.rados.rados.0.smithi033.stdout:861: finishing write tid 3 to smithi03385152-13 2024-09-21T09:53:51.197 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 605 (ObjNum 292 snap 71 seq_num 292) dirty exists 2024-09-21T09:53:51.197 INFO:tasks.rados.rados.0.smithi033.stdout:861: left oid 13 (ObjNum 292 snap 71 seq_num 292) 2024-09-21T09:53:51.214 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.945+0000 7f19d3c02640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.214 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.945+0000 7f19d3c02640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.215 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.945+0000 7f19d3c02640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.215 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.945+0000 7f19d11ff640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.215 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.945+0000 7f19d11ff640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.215 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.945+0000 7f19d11ff640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.215 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.945+0000 7f19d11ff640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.215 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.945+0000 7f19d11ff640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.216 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.947+0000 7f19d3c02640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.216 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:50.947+0000 7f19d3c02640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:53:51.495 INFO:tasks.rados.rados.0.smithi033.stdout:861: done (2 left) 2024-09-21T09:53:51.495 INFO:tasks.rados.rados.0.smithi033.stdout:863: done (1 left) 2024-09-21T09:53:51.495 INFO:tasks.rados.rados.0.smithi033.stdout:865: done (0 left) 2024-09-21T09:53:51.495 INFO:tasks.rados.rados.0.smithi033.stdout:866: read oid 46 snap -1 2024-09-21T09:53:51.495 INFO:tasks.rados.rados.0.smithi033.stdout:866: expect (ObjNum 253 snap 65 seq_num 253) 2024-09-21T09:53:51.495 INFO:tasks.rados.rados.0.smithi033.stdout:867: read oid 12 snap -1 2024-09-21T09:53:51.495 INFO:tasks.rados.rados.0.smithi033.stdout:867: expect (ObjNum 262 snap 67 seq_num 262) 2024-09-21T09:53:51.495 INFO:tasks.rados.rados.0.smithi033.stdout:868: read oid 29 snap -1 2024-09-21T09:53:51.496 INFO:tasks.rados.rados.0.smithi033.stdout:868: expect deleted 2024-09-21T09:53:51.496 INFO:tasks.rados.rados.0.smithi033.stdout:869: snap_remove snap 70 2024-09-21T09:53:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:51] ENGINE Bus STARTING 2024-09-21T09:53:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:51] ENGINE Serving on http://:::9283 2024-09-21T09:53:51.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:53:51] ENGINE Bus STARTED 2024-09-21T09:53:51.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:51.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:51.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:51.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: mgrmap e31: y(active, since 1.01099s) 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[27969]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:51.933 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:51.934 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:51.934 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: mgrmap e31: y(active, since 1.01099s) 2024-09-21T09:53:51.934 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:53:51.934 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:51 smithi033 ceph-mon[39045]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: mgrmap e31: y(active, since 1.01099s) 2024-09-21T09:53:51.941 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: osdmap e215: 8 total, 8 up, 8 in 2024-09-21T09:53:51.942 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:51 smithi105 ceph-mon[31286]: osdmap e216: 8 total, 8 up, 8 in 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:52.173+0000 7f19b798d640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:53:52] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: Traceback (most recent call last): 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: self._connections.run(self.expiration_interval) 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: self._run(expiration_interval) 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:52.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: s = self.context.wrap_socket( 2024-09-21T09:53:52.248 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:52.248 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: return self.sslsocket_class._create( 2024-09-21T09:53:52.248 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:52.248 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: self.do_handshake() 2024-09-21T09:53:52.248 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:52.248 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: self._sslobj.do_handshake() 2024-09-21T09:53:52.248 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:52.248 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:53:52 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:53:52.502 INFO:tasks.rados.rados.0.smithi033.stdout:869: done (3 left) 2024-09-21T09:53:52.502 INFO:tasks.rados.rados.0.smithi033.stdout:870: setattr oid 42 current snap is 72 2024-09-21T09:53:52.502 INFO:tasks.rados.rados.0.smithi033.stdout:866: expect (ObjNum 253 snap 65 seq_num 253) 2024-09-21T09:53:52.554 INFO:tasks.rados.rados.0.smithi033.stdout:867: expect (ObjNum 262 snap 67 seq_num 262) 2024-09-21T09:53:52.898 INFO:tasks.rados.rados.0.smithi033.stdout:866: done (3 left) 2024-09-21T09:53:52.898 INFO:tasks.rados.rados.0.smithi033.stdout:867: done (2 left) 2024-09-21T09:53:52.898 INFO:tasks.rados.rados.0.smithi033.stdout:868: done (1 left) 2024-09-21T09:53:52.898 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 72 2024-09-21T09:53:52.898 INFO:tasks.rados.rados.0.smithi033.stdout:871: seq_num 294 ranges {5611520=131072} 2024-09-21T09:53:52.929 INFO:tasks.rados.rados.0.smithi033.stdout:871: writing smithi03385152-12 from 5611520 to 5742592 tid 1 2024-09-21T09:53:52.929 INFO:tasks.rados.rados.0.smithi033.stdout:872: read oid 21 snap 68 2024-09-21T09:53:52.929 INFO:tasks.rados.rados.0.smithi033.stdout:872: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:53:52.929 INFO:tasks.rados.rados.0.smithi033.stdout:873: read oid 50 snap -1 2024-09-21T09:53:52.929 INFO:tasks.rados.rados.0.smithi033.stdout:873: expect deleted 2024-09-21T09:53:52.929 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 72 2024-09-21T09:53:52.929 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 599 (ObjNum 284 snap 69 seq_num 284) dirty exists 2024-09-21T09:53:52.929 INFO:tasks.rados.rados.0.smithi033.stdout:874: seq_num 295 ranges {2252800=147456} 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:874: writing smithi03385152-32 from 2252800 to 2400256 tid 1 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:870: done (4 left) 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:875: read oid 9 snap 68 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:875: expect deleted 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:876: read oid 20 snap -1 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:876: expect deleted 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:877: snap_create 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:871: finishing write tid 1 to smithi03385152-12 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:871: finishing write tid 2 to smithi03385152-12 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:871: finishing write tid 3 to smithi03385152-12 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 368 (ObjNum 294 snap 72 seq_num 294) dirty exists 2024-09-21T09:53:52.943 INFO:tasks.rados.rados.0.smithi033.stdout:871: left oid 12 (ObjNum 294 snap 72 seq_num 294) 2024-09-21T09:53:52.944 INFO:tasks.rados.rados.0.smithi033.stdout:872: expect (ObjNum 112 snap 25 seq_num 112) 2024-09-21T09:53:53.074 INFO:tasks.rados.rados.0.smithi033.stdout:874: finishing write tid 1 to smithi03385152-32 2024-09-21T09:53:53.074 INFO:tasks.rados.rados.0.smithi033.stdout:874: finishing write tid 2 to smithi03385152-32 2024-09-21T09:53:53.074 INFO:tasks.rados.rados.0.smithi033.stdout:874: finishing write tid 3 to smithi03385152-32 2024-09-21T09:53:53.074 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 99 (ObjNum 295 snap 72 seq_num 295) dirty exists 2024-09-21T09:53:53.074 INFO:tasks.rados.rados.0.smithi033.stdout:874: left oid 32 (ObjNum 295 snap 72 seq_num 295) 2024-09-21T09:53:53.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[39045]: [21/Sep/2024:09:53:52] ENGINE Bus STARTING 2024-09-21T09:53:53.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[39045]: [21/Sep/2024:09:53:52] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:53:53.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[39045]: [21/Sep/2024:09:53:52] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:53.496 INFO:journalctl@ceph.mon.c.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:53:53.496 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:53.496 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:53.496 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: self.do_handshake() 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:53.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[39045]: pgmap v6: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 644 MiB/s rd, 354 MiB/s wr, 3.68k op/s; 582 KiB/s, 2 objects/s recovering 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[39045]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[39045]: mgrmap e32: y(active, since 2s) 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[27969]: [21/Sep/2024:09:53:52] ENGINE Bus STARTING 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[27969]: [21/Sep/2024:09:53:52] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[27969]: [21/Sep/2024:09:53:52] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:53.498 INFO:journalctl@ceph.mon.a.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: self.do_handshake() 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[27969]: pgmap v6: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 644 MiB/s rd, 354 MiB/s wr, 3.68k op/s; 582 KiB/s, 2 objects/s recovering 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[27969]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:53:53.499 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:53 smithi033 ceph-mon[27969]: mgrmap e32: y(active, since 2s) 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:53 smithi105 ceph-mon[31286]: [21/Sep/2024:09:53:52] ENGINE Bus STARTING 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:53 smithi105 ceph-mon[31286]: [21/Sep/2024:09:53:52] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:53 smithi105 ceph-mon[31286]: [21/Sep/2024:09:53:52] ENGINE Error in HTTPServer.serve 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-21T09:53:53.530 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:53 smithi105 ceph-mon[31286]: pgmap v6: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 644 MiB/s rd, 354 MiB/s wr, 3.68k op/s; 582 KiB/s, 2 objects/s recovering 2024-09-21T09:53:53.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:53 smithi105 ceph-mon[31286]: osdmap e217: 8 total, 8 up, 8 in 2024-09-21T09:53:53.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:53 smithi105 ceph-mon[31286]: mgrmap e32: y(active, since 2s) 2024-09-21T09:53:53.688 INFO:tasks.rados.rados.0.smithi033.stdout:871: done (6 left) 2024-09-21T09:53:53.688 INFO:tasks.rados.rados.0.smithi033.stdout:872: done (5 left) 2024-09-21T09:53:53.688 INFO:tasks.rados.rados.0.smithi033.stdout:873: done (4 left) 2024-09-21T09:53:53.688 INFO:tasks.rados.rados.0.smithi033.stdout:874: done (3 left) 2024-09-21T09:53:53.688 INFO:tasks.rados.rados.0.smithi033.stdout:875: done (2 left) 2024-09-21T09:53:53.688 INFO:tasks.rados.rados.0.smithi033.stdout:876: done (1 left) 2024-09-21T09:53:53.688 INFO:tasks.rados.rados.0.smithi033.stdout:877: done (0 left) 2024-09-21T09:53:53.688 INFO:tasks.rados.rados.0.smithi033.stdout:878: snap_remove snap 71 2024-09-21T09:53:54.486 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:54 smithi033 ceph-mon[27969]: [21/Sep/2024:09:53:52] ENGINE Serving on http://172.21.15.33:8765 2024-09-21T09:53:54.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:54 smithi033 ceph-mon[27969]: [21/Sep/2024:09:53:52] ENGINE Bus STARTED 2024-09-21T09:53:54.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:54 smithi033 ceph-mon[27969]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:53:54.487 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:54 smithi033 ceph-mon[27969]: pgmap v9: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 213 MiB/s rd, 117 MiB/s wr, 1.22k op/s; 193 KiB/s, 0 objects/s recovering 2024-09-21T09:53:54.487 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:54 smithi033 ceph-mon[39045]: [21/Sep/2024:09:53:52] ENGINE Serving on http://172.21.15.33:8765 2024-09-21T09:53:54.487 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:54 smithi033 ceph-mon[39045]: [21/Sep/2024:09:53:52] ENGINE Bus STARTED 2024-09-21T09:53:54.487 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:54 smithi033 ceph-mon[39045]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:53:54.487 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:54 smithi033 ceph-mon[39045]: pgmap v9: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 213 MiB/s rd, 117 MiB/s wr, 1.22k op/s; 193 KiB/s, 0 objects/s recovering 2024-09-21T09:53:54.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:54 smithi105 ceph-mon[31286]: [21/Sep/2024:09:53:52] ENGINE Serving on http://172.21.15.33:8765 2024-09-21T09:53:54.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:54 smithi105 ceph-mon[31286]: [21/Sep/2024:09:53:52] ENGINE Bus STARTED 2024-09-21T09:53:54.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:54 smithi105 ceph-mon[31286]: osdmap e218: 8 total, 8 up, 8 in 2024-09-21T09:53:54.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:54 smithi105 ceph-mon[31286]: pgmap v9: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 213 MiB/s rd, 117 MiB/s wr, 1.22k op/s; 193 KiB/s, 0 objects/s recovering 2024-09-21T09:53:54.696 INFO:tasks.rados.rados.0.smithi033.stdout:878: done (0 left) 2024-09-21T09:53:54.696 INFO:tasks.rados.rados.0.smithi033.stdout:879: rmattr oid 41 current snap is 73 2024-09-21T09:53:54.696 INFO:tasks.rados.rados.0.smithi033.stdout:879: done (0 left) 2024-09-21T09:53:54.696 INFO:tasks.rados.rados.0.smithi033.stdout:880: snap_remove snap 72 2024-09-21T09:53:55.699 INFO:tasks.rados.rados.0.smithi033.stdout:880: done (0 left) 2024-09-21T09:53:55.699 INFO:tasks.rados.rados.0.smithi033.stdout:881: setattr oid 11 current snap is 73 2024-09-21T09:53:55.700 INFO:tasks.rados.rados.0.smithi033.stdout:882: rmattr oid 31 current snap is 73 2024-09-21T09:53:55.701 INFO:tasks.rados.rados.0.smithi033.stdout:882: done (1 left) 2024-09-21T09:53:55.701 INFO:tasks.rados.rados.0.smithi033.stdout:883: rollback oid 42 current snap is 73 2024-09-21T09:53:55.701 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 42 to 68 2024-09-21T09:53:55.701 INFO:tasks.rados.rados.0.smithi033.stdout:884: rollback oid 38 current snap is 73 2024-09-21T09:53:55.701 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 68 2024-09-21T09:53:55.701 INFO:tasks.rados.rados.0.smithi033.stdout:885: snap_create 2024-09-21T09:53:55.701 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:53:55] ENGINE Bus STARTING 2024-09-21T09:53:55.701 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: CherryPy Checker: 2024-09-21T09:53:55.701 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: The Application mounted at '' has an empty config. 2024-09-21T09:53:55.701 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: 2024-09-21T09:53:55.702 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:53:55] ENGINE Serving on http://:::9283 2024-09-21T09:53:55.702 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: [21/Sep/2024:09:53:55] ENGINE Bus STARTED 2024-09-21T09:53:55.707 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 361 (ObjNum 192 snap 45 seq_num 192) dirty exists 2024-09-21T09:53:55.709 INFO:tasks.rados.rados.0.smithi033.stdout:883: finishing rollback tid 0 to smithi03385152-42 2024-09-21T09:53:55.710 INFO:tasks.rados.rados.0.smithi033.stdout:883: finishing rollback tid 1 to smithi03385152-42 2024-09-21T09:53:55.710 INFO:tasks.rados.rados.0.smithi033.stdout:883: finishing rollback tid 2 to smithi03385152-42 2024-09-21T09:53:55.710 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 602 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-21T09:53:55.710 INFO:tasks.rados.rados.0.smithi033.stdout:884: finishing rollback tid 0 to smithi03385152-38 2024-09-21T09:53:55.710 INFO:tasks.rados.rados.0.smithi033.stdout:884: finishing rollback tid 1 to smithi03385152-38 2024-09-21T09:53:55.710 INFO:tasks.rados.rados.0.smithi033.stdout:884: finishing rollback tid 2 to smithi03385152-38 2024-09-21T09:53:55.711 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 605 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:53:55.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: mgrmap e33: y(active, since 4s) 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: Standby manager daemon x started 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:55.957 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: mgrmap e33: y(active, since 4s) 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: Standby manager daemon x started 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:55.958 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:55 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: osdmap e219: 8 total, 8 up, 8 in 2024-09-21T09:53:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: mgrmap e33: y(active, since 4s) 2024-09-21T09:53:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: Standby manager daemon x started 2024-09-21T09:53:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:53:56.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:53:56.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:53:56.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/1000794869' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:53:56.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:55 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.711 INFO:tasks.rados.rados.0.smithi033.stdout:881: done (3 left) 2024-09-21T09:53:56.711 INFO:tasks.rados.rados.0.smithi033.stdout:883: done (2 left) 2024-09-21T09:53:56.711 INFO:tasks.rados.rados.0.smithi033.stdout:884: done (1 left) 2024-09-21T09:53:56.711 INFO:tasks.rados.rados.0.smithi033.stdout:885: done (0 left) 2024-09-21T09:53:56.711 INFO:tasks.rados.rados.0.smithi033.stdout:886: rollback oid 43 current snap is 74 2024-09-21T09:53:56.711 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 43 to 68 2024-09-21T09:53:56.711 INFO:tasks.rados.rados.0.smithi033.stdout:887: snap_create 2024-09-21T09:53:56.715 INFO:tasks.rados.rados.0.smithi033.stdout:886: finishing rollback tid 0 to smithi03385152-43 2024-09-21T09:53:56.716 INFO:tasks.rados.rados.0.smithi033.stdout:886: finishing rollback tid 1 to smithi03385152-43 2024-09-21T09:53:56.717 INFO:tasks.rados.rados.0.smithi033.stdout:886: finishing rollback tid 2 to smithi03385152-43 2024-09-21T09:53:56.717 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 364 (ObjNum 266 snap 67 seq_num 266) dirty exists 2024-09-21T09:53:56.955 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[39045]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:53:56.955 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[39045]: mgrmap e34: y(active, since 5s), standbys: x 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[39045]: pgmap v12: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[27969]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[27969]: mgrmap e34: y(active, since 5s), standbys: x 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:56.956 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[27969]: pgmap v12: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:56.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:56.957 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:56 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:57.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:53:56.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:53:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-mon[31286]: osdmap e220: 8 total, 8 up, 8 in 2024-09-21T09:53:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-mon[31286]: mgrmap e34: y(active, since 5s), standbys: x 2024-09-21T09:53:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:53:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-mon[31286]: pgmap v12: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:53:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:57.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:56 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:53:57.712 INFO:tasks.rados.rados.0.smithi033.stdout:886: done (1 left) 2024-09-21T09:53:57.712 INFO:tasks.rados.rados.0.smithi033.stdout:887: done (0 left) 2024-09-21T09:53:57.712 INFO:tasks.rados.rados.0.smithi033.stdout:888: rmattr oid 29 current snap is 75 2024-09-21T09:53:57.713 INFO:tasks.rados.rados.0.smithi033.stdout:888: done (0 left) 2024-09-21T09:53:57.713 INFO:tasks.rados.rados.0.smithi033.stdout:889: snap_create 2024-09-21T09:53:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:57 smithi033 ceph-mon[39045]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:53:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:57 smithi033 ceph-mon[39045]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:53:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:57 smithi033 ceph-mon[27969]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:53:57.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:57 smithi033 ceph-mon[27969]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:53:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:57 smithi105 ceph-mon[31286]: osdmap e221: 8 total, 8 up, 8 in 2024-09-21T09:53:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:57 smithi105 ceph-mon[31286]: osdmap e222: 8 total, 8 up, 8 in 2024-09-21T09:53:58.711 INFO:tasks.rados.rados.0.smithi033.stdout:889: done (0 left) 2024-09-21T09:53:58.711 INFO:tasks.rados.rados.0.smithi033.stdout:890: delete oid 33 current snap is 76 2024-09-21T09:53:58.713 INFO:tasks.rados.rados.0.smithi033.stdout:890: done (0 left) 2024-09-21T09:53:58.713 INFO:tasks.rados.rados.0.smithi033.stdout:891: setattr oid 48 current snap is 76 2024-09-21T09:53:58.716 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 76 2024-09-21T09:53:58.716 INFO:tasks.rados.rados.0.smithi033.stdout:892: seq_num 297 ranges {1425408=180224} 2024-09-21T09:53:58.725 INFO:tasks.rados.rados.0.smithi033.stdout:892: writing smithi03385152-2 from 1425408 to 1605632 tid 1 2024-09-21T09:53:58.725 INFO:tasks.rados.rados.0.smithi033.stdout:893: read oid 37 snap -1 2024-09-21T09:53:58.725 INFO:tasks.rados.rados.0.smithi033.stdout:893: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:53:58.726 INFO:tasks.rados.rados.0.smithi033.stdout:894: copy_from oid 40 from oid 33 current snap is 76 2024-09-21T09:53:58.726 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 578 (ObjNum 222 snap 52 seq_num 222) dirty exists 2024-09-21T09:53:58.726 INFO:tasks.rados.rados.0.smithi033.stdout:891: done (3 left) 2024-09-21T09:53:58.726 INFO:tasks.rados.rados.0.smithi033.stdout:895: snap_remove snap 68 2024-09-21T09:53:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:58 smithi033 ceph-mon[39045]: pgmap v15: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 68 KiB/s wr, 33 op/s 2024-09-21T09:53:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:58 smithi033 ceph-mon[39045]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:53:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:58 smithi033 ceph-mon[27969]: pgmap v15: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 68 KiB/s wr, 33 op/s 2024-09-21T09:53:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:58 smithi033 ceph-mon[27969]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:53:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:58 smithi105 ceph-mon[31286]: pgmap v15: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 68 KiB/s wr, 33 op/s 2024-09-21T09:53:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:58 smithi105 ceph-mon[31286]: osdmap e223: 8 total, 8 up, 8 in 2024-09-21T09:53:59.733 INFO:tasks.rados.rados.0.smithi033.stdout:895: done (3 left) 2024-09-21T09:53:59.733 INFO:tasks.rados.rados.0.smithi033.stdout:896: setattr oid 27 current snap is 76 2024-09-21T09:53:59.733 INFO:tasks.rados.rados.0.smithi033.stdout:894: finishing copy_from to smithi03385152-40 2024-09-21T09:53:59.733 INFO:tasks.rados.rados.0.smithi033.stdout:894: got expected ENOENT (src dne) 2024-09-21T09:53:59.733 INFO:tasks.rados.rados.0.smithi033.stdout:894: finishing copy_from racing read to smithi03385152-40 2024-09-21T09:53:59.733 INFO:tasks.rados.rados.0.smithi033.stdout:892: finishing write tid 1 to smithi03385152-2 2024-09-21T09:53:59.733 INFO:tasks.rados.rados.0.smithi033.stdout:892: finishing write tid 2 to smithi03385152-2 2024-09-21T09:53:59.734 INFO:tasks.rados.rados.0.smithi033.stdout:892: finishing write tid 3 to smithi03385152-2 2024-09-21T09:53:59.734 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 188 (ObjNum 297 snap 76 seq_num 297) dirty exists 2024-09-21T09:53:59.734 INFO:tasks.rados.rados.0.smithi033.stdout:892: left oid 2 (ObjNum 297 snap 76 seq_num 297) 2024-09-21T09:53:59.734 INFO:tasks.rados.rados.0.smithi033.stdout:893: expect (ObjNum 174 snap 38 seq_num 174) 2024-09-21T09:54:00.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:00.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:53:59 smithi105 ceph-mon[31286]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:54:00.035 INFO:tasks.rados.rados.0.smithi033.stdout:892: done (3 left) 2024-09-21T09:54:00.035 INFO:tasks.rados.rados.0.smithi033.stdout:893: done (2 left) 2024-09-21T09:54:00.035 INFO:tasks.rados.rados.0.smithi033.stdout:894: done (1 left) 2024-09-21T09:54:00.035 INFO:tasks.rados.rados.0.smithi033.stdout:897: snap_create 2024-09-21T09:54:00.038 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 316 (ObjNum 290 snap 70 seq_num 290) dirty exists 2024-09-21T09:54:00.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[27969]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:54:00.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:00.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:00.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:00.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:53:59 smithi033 ceph-mon[39045]: osdmap e224: 8 total, 8 up, 8 in 2024-09-21T09:54:00.410 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:00.776 INFO:tasks.rados.rados.0.smithi033.stdout:896: done (1 left) 2024-09-21T09:54:00.776 INFO:tasks.rados.rados.0.smithi033.stdout:897: done (0 left) 2024-09-21T09:54:00.776 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 77 2024-09-21T09:54:00.777 INFO:tasks.rados.rados.0.smithi033.stdout:898: seq_num 298 ranges {4825088=196608} 2024-09-21T09:54:00.803 INFO:tasks.rados.rados.0.smithi033.stdout:898: writing smithi03385152-37 from 4825088 to 5021696 tid 1 2024-09-21T09:54:00.804 INFO:tasks.rados.rados.0.smithi033.stdout:899: snap_create 2024-09-21T09:54:00.809 INFO:tasks.rados.rados.0.smithi033.stdout:898: finishing write tid 1 to smithi03385152-37 2024-09-21T09:54:00.809 INFO:tasks.rados.rados.0.smithi033.stdout:898: finishing write tid 2 to smithi03385152-37 2024-09-21T09:54:00.810 INFO:tasks.rados.rados.0.smithi033.stdout:898: finishing write tid 3 to smithi03385152-37 2024-09-21T09:54:00.810 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 366 (ObjNum 298 snap 77 seq_num 298) dirty exists 2024-09-21T09:54:00.810 INFO:tasks.rados.rados.0.smithi033.stdout:898: left oid 37 (ObjNum 298 snap 77 seq_num 298) 2024-09-21T09:54:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:00 smithi105 ceph-mon[31286]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:54:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:00 smithi105 ceph-mon[31286]: Reconfiguring daemon prometheus.a on smithi105 2024-09-21T09:54:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:00 smithi105 ceph-mon[31286]: pgmap v18: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 68 KiB/s wr, 33 op/s 2024-09-21T09:54:01.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:00 smithi033 ceph-mon[39045]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:54:01.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:00 smithi033 ceph-mon[39045]: Reconfiguring daemon prometheus.a on smithi105 2024-09-21T09:54:01.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:00 smithi033 ceph-mon[39045]: pgmap v18: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 68 KiB/s wr, 33 op/s 2024-09-21T09:54:01.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:00 smithi033 ceph-mon[27969]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-21T09:54:01.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:00 smithi033 ceph-mon[27969]: Reconfiguring daemon prometheus.a on smithi105 2024-09-21T09:54:01.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:00 smithi033 ceph-mon[27969]: pgmap v18: 121 pgs: 121 active+clean; 158 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 8.0 MiB/s rd, 68 KiB/s wr, 33 op/s 2024-09-21T09:54:01.497 INFO:tasks.rados.rados.0.smithi033.stdout:898: done (1 left) 2024-09-21T09:54:01.497 INFO:tasks.rados.rados.0.smithi033.stdout:899: done (0 left) 2024-09-21T09:54:01.497 INFO:tasks.rados.rados.0.smithi033.stdout:900: read oid 11 snap 73 2024-09-21T09:54:01.497 INFO:tasks.rados.rados.0.smithi033.stdout:900: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:54:01.498 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 78 2024-09-21T09:54:01.498 INFO:tasks.rados.rados.0.smithi033.stdout:901: seq_num 299 ranges {3678208=212992} 2024-09-21T09:54:01.519 INFO:tasks.rados.rados.0.smithi033.stdout:901: writing smithi03385152-21 from 3678208 to 3891200 tid 1 2024-09-21T09:54:01.519 INFO:tasks.rados.rados.0.smithi033.stdout:902: delete oid 47 current snap is 78 2024-09-21T09:54:01.519 INFO:tasks.rados.rados.0.smithi033.stdout:900: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:54:01.678 INFO:tasks.rados.rados.0.smithi033.stdout:901: finishing write tid 1 to smithi03385152-21 2024-09-21T09:54:01.678 INFO:tasks.rados.rados.0.smithi033.stdout:901: finishing write tid 2 to smithi03385152-21 2024-09-21T09:54:01.678 INFO:tasks.rados.rados.0.smithi033.stdout:901: finishing write tid 3 to smithi03385152-21 2024-09-21T09:54:01.678 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 351 (ObjNum 299 snap 78 seq_num 299) dirty exists 2024-09-21T09:54:01.678 INFO:tasks.rados.rados.0.smithi033.stdout:901: left oid 21 (ObjNum 299 snap 78 seq_num 299) 2024-09-21T09:54:01.678 INFO:tasks.rados.rados.0.smithi033.stdout:900: done (2 left) 2024-09-21T09:54:01.678 INFO:tasks.rados.rados.0.smithi033.stdout:901: done (1 left) 2024-09-21T09:54:01.678 INFO:tasks.rados.rados.0.smithi033.stdout:902: done (0 left) 2024-09-21T09:54:01.679 INFO:tasks.rados.rados.0.smithi033.stdout:903: snap_create 2024-09-21T09:54:01.860 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[67120]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:54:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.51.0" 2024-09-21T09:54:01.860 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:54:01.860 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:01.861 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:01.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:01 smithi105 ceph-mon[31286]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:54:02.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:54:02.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:02.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:54:02.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:02.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:02.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:02.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:02.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:02.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:02.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[27969]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:54:02.251 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: osdmap e225: 8 total, 8 up, 8 in 2024-09-21T09:54:02.251 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:02.251 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:54:02.252 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.252 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.252 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.252 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.252 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.252 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.252 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.252 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.253 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.253 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:02.253 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:02.253 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.253 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.253 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.253 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:02.254 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:02.255 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:01 smithi033 ceph-mon[39045]: osdmap e226: 8 total, 8 up, 8 in 2024-09-21T09:54:02.513 INFO:tasks.rados.rados.0.smithi033.stdout:903: done (0 left) 2024-09-21T09:54:02.513 INFO:tasks.rados.rados.0.smithi033.stdout:904: setattr oid 39 current snap is 79 2024-09-21T09:54:02.514 INFO:tasks.rados.rados.0.smithi033.stdout:905: read oid 37 snap -1 2024-09-21T09:54:02.515 INFO:tasks.rados.rados.0.smithi033.stdout:905: expect (ObjNum 298 snap 77 seq_num 298) 2024-09-21T09:54:02.515 INFO:tasks.rados.rados.0.smithi033.stdout:906: read oid 25 snap -1 2024-09-21T09:54:02.515 INFO:tasks.rados.rados.0.smithi033.stdout:906: expect (ObjNum 243 snap 63 seq_num 243) 2024-09-21T09:54:02.515 INFO:tasks.rados.rados.0.smithi033.stdout:907: read oid 43 snap -1 2024-09-21T09:54:02.515 INFO:tasks.rados.rados.0.smithi033.stdout:907: expect (ObjNum 266 snap 67 seq_num 266) 2024-09-21T09:54:02.515 INFO:tasks.rados.rados.0.smithi033.stdout:908: snap_remove snap 77 2024-09-21T09:54:02.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:02 smithi105 ceph-mon[31286]: pgmap v21: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 92 KiB/s wr, 24 op/s 2024-09-21T09:54:02.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:02 smithi105 ceph-mon[31286]: mgrmap e35: y(active, since 12s), standbys: x 2024-09-21T09:54:02.862 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:02 smithi105 ceph-mon[31286]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:54:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:02 smithi033 ceph-mon[27969]: pgmap v21: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 92 KiB/s wr, 24 op/s 2024-09-21T09:54:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:02 smithi033 ceph-mon[27969]: mgrmap e35: y(active, since 12s), standbys: x 2024-09-21T09:54:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:02 smithi033 ceph-mon[27969]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:54:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:02 smithi033 ceph-mon[39045]: pgmap v21: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 92 KiB/s wr, 24 op/s 2024-09-21T09:54:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:02 smithi033 ceph-mon[39045]: mgrmap e35: y(active, since 12s), standbys: x 2024-09-21T09:54:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:02 smithi033 ceph-mon[39045]: osdmap e227: 8 total, 8 up, 8 in 2024-09-21T09:54:03.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 systemd[1]: Stopping Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:03.516 INFO:tasks.rados.rados.0.smithi033.stdout:908: done (4 left) 2024-09-21T09:54:03.516 INFO:tasks.rados.rados.0.smithi033.stdout:909: setattr oid 26 current snap is 79 2024-09-21T09:54:03.516 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 89 (ObjNum 199 snap 47 seq_num 199) dirty exists 2024-09-21T09:54:03.516 INFO:tasks.rados.rados.0.smithi033.stdout:906: expect (ObjNum 243 snap 63 seq_num 243) 2024-09-21T09:54:03.642 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T09:54:03.642 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.350Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.351Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.351Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[75764]: ts=2024-09-21T09:54:03.351Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-21T09:54:03.643 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 podman[78627]: 2024-09-21 09:54:03.360672123 +0000 UTC m=+0.133760392 container died 12975b5508f873726f79426f72b84cdd18bb11bae88e5260c433d43acdec14fa (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:03.713 INFO:tasks.rados.rados.0.smithi033.stdout:905: expect (ObjNum 298 snap 77 seq_num 298) 2024-09-21T09:54:04.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 podman[78627]: 2024-09-21 09:54:03.748290678 +0000 UTC m=+0.521378937 container cleanup 12975b5508f873726f79426f72b84cdd18bb11bae88e5260c433d43acdec14fa (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:04.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:03 smithi105 bash[78627]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T09:54:04.094 INFO:tasks.rados.rados.0.smithi033.stdout:907: expect (ObjNum 266 snap 67 seq_num 266) 2024-09-21T09:54:04.402 INFO:tasks.rados.rados.0.smithi033.stdout:904: done (4 left) 2024-09-21T09:54:04.402 INFO:tasks.rados.rados.0.smithi033.stdout:905: done (3 left) 2024-09-21T09:54:04.402 INFO:tasks.rados.rados.0.smithi033.stdout:906: done (2 left) 2024-09-21T09:54:04.402 INFO:tasks.rados.rados.0.smithi033.stdout:907: done (1 left) 2024-09-21T09:54:04.402 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 79 2024-09-21T09:54:04.402 INFO:tasks.rados.rados.0.smithi033.stdout:910: seq_num 301 ranges {0=245760} 2024-09-21T09:54:04.403 INFO:tasks.rados.rados.0.smithi033.stdout:910: writing smithi03385152-33 from 0 to 245760 tid 1 2024-09-21T09:54:04.403 INFO:tasks.rados.rados.0.smithi033.stdout:911: delete oid 43 current snap is 79 2024-09-21T09:54:04.404 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 378 (ObjNum 272 snap 69 seq_num 272) dirty exists 2024-09-21T09:54:04.405 INFO:tasks.rados.rados.0.smithi033.stdout:909: done (2 left) 2024-09-21T09:54:04.406 INFO:tasks.rados.rados.0.smithi033.stdout:911: done (1 left) 2024-09-21T09:54:04.406 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 79 2024-09-21T09:54:04.406 INFO:tasks.rados.rados.0.smithi033.stdout:912: seq_num 303 ranges {1327104=278528} 2024-09-21T09:54:04.414 INFO:tasks.rados.rados.0.smithi033.stdout:912: writing smithi03385152-48 from 1327104 to 1605632 tid 1 2024-09-21T09:54:04.415 INFO:tasks.rados.rados.0.smithi033.stdout:913: snap_create 2024-09-21T09:54:04.415 INFO:tasks.rados.rados.0.smithi033.stdout:910: finishing write tid 1 to smithi03385152-33 2024-09-21T09:54:04.415 INFO:tasks.rados.rados.0.smithi033.stdout:910: finishing write tid 2 to smithi03385152-33 2024-09-21T09:54:04.415 INFO:tasks.rados.rados.0.smithi033.stdout:910: finishing write tid 3 to smithi03385152-33 2024-09-21T09:54:04.415 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 318 (ObjNum 301 snap 79 seq_num 301) dirty exists 2024-09-21T09:54:04.415 INFO:tasks.rados.rados.0.smithi033.stdout:910: left oid 33 (ObjNum 301 snap 79 seq_num 301) 2024-09-21T09:54:04.418 INFO:tasks.rados.rados.0.smithi033.stdout:912: finishing write tid 1 to smithi03385152-48 2024-09-21T09:54:04.418 INFO:tasks.rados.rados.0.smithi033.stdout:912: finishing write tid 2 to smithi03385152-48 2024-09-21T09:54:04.420 INFO:tasks.rados.rados.0.smithi033.stdout:912: finishing write tid 3 to smithi03385152-48 2024-09-21T09:54:04.420 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 589 (ObjNum 303 snap 79 seq_num 303) dirty exists 2024-09-21T09:54:04.420 INFO:tasks.rados.rados.0.smithi033.stdout:912: left oid 48 (ObjNum 303 snap 79 seq_num 303) 2024-09-21T09:54:04.429 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 podman[78647]: 2024-09-21 09:54:04.057974131 +0000 UTC m=+0.698864638 container remove 12975b5508f873726f79426f72b84cdd18bb11bae88e5260c433d43acdec14fa (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:04.429 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Deactivated successfully. 2024-09-21T09:54:04.430 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 systemd[1]: Stopped Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:04.430 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a.service: Consumed 1.139s CPU time. 2024-09-21T09:54:04.430 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 systemd[1]: Starting Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:04.518 INFO:tasks.rados.rados.0.smithi033.stdout:910: done (2 left) 2024-09-21T09:54:04.519 INFO:tasks.rados.rados.0.smithi033.stdout:912: done (1 left) 2024-09-21T09:54:04.519 INFO:tasks.rados.rados.0.smithi033.stdout:913: done (0 left) 2024-09-21T09:54:04.519 INFO:tasks.rados.rados.0.smithi033.stdout:914: snap_remove snap 76 2024-09-21T09:54:04.704 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-mon[31286]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:54:04.704 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-mon[31286]: pgmap v24: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 92 KiB/s wr, 24 op/s 2024-09-21T09:54:04.704 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 podman[78743]: 2024-09-21 09:54:04.331964925 +0000 UTC m=+0.016778267 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-21T09:54:04.705 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 podman[78743]: 2024-09-21 09:54:04.459346059 +0000 UTC m=+0.144159437 container create 670cdb3edd9d943943f96657260d47591a5d85802bcd87a16e847142b4dd80fe (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:04 smithi033 ceph-mon[39045]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:54:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:04 smithi033 ceph-mon[39045]: pgmap v24: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 92 KiB/s wr, 24 op/s 2024-09-21T09:54:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:04 smithi033 ceph-mon[27969]: osdmap e228: 8 total, 8 up, 8 in 2024-09-21T09:54:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:04 smithi033 ceph-mon[27969]: pgmap v24: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 92 KiB/s wr, 24 op/s 2024-09-21T09:54:05.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 podman[78743]: 2024-09-21 09:54:04.703222153 +0000 UTC m=+0.388035488 container init 670cdb3edd9d943943f96657260d47591a5d85802bcd87a16e847142b4dd80fe (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:05.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 podman[78743]: 2024-09-21 09:54:04.708553867 +0000 UTC m=+0.393367199 container start 670cdb3edd9d943943f96657260d47591a5d85802bcd87a16e847142b4dd80fe (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T09:54:05.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.727Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.727Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.727Z caller=main.go:623 level=info host_details="(Linux 5.14.0-508.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024 x86_64 smithi105 (none))" 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.727Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.727Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.729Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.729Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.731Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.731Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.733Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.733Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.26µs 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.733Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.733Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.746Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-21T09:54:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.752Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.752Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.752Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=34.214µs wal_replay_duration=19.616916ms wbl_replay_duration=179ns total_replay_duration=19.678772ms 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.756Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.756Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.756Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.770Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.462265ms db_storage=790ns remote_storage=1.189µs web_handler=406ns query_engine=568ns scrape=412.791µs scrape_sd=97.329µs notify=10.4µs notify_sd=10.136µs rules=13.579102ms tracing=5.358µs 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.770Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:04.770Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 bash[78743]: 670cdb3edd9d943943f96657260d47591a5d85802bcd87a16e847142b4dd80fe 2024-09-21T09:54:05.032 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:04 smithi105 systemd[1]: Started Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:05.527 INFO:tasks.rados.rados.0.smithi033.stdout:914: done (0 left) 2024-09-21T09:54:05.527 INFO:tasks.rados.rados.0.smithi033.stdout:915: rmattr oid 46 current snap is 80 2024-09-21T09:54:05.527 INFO:tasks.rados.rados.0.smithi033.stdout:915: done (0 left) 2024-09-21T09:54:05.527 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 80 2024-09-21T09:54:05.527 INFO:tasks.rados.rados.0.smithi033.stdout:916: seq_num 304 ranges {3891200=294912} 2024-09-21T09:54:05.550 INFO:tasks.rados.rados.0.smithi033.stdout:916: writing smithi03385152-21 from 3891200 to 4186112 tid 1 2024-09-21T09:54:05.550 INFO:tasks.rados.rados.0.smithi033.stdout:917: delete oid 39 current snap is 80 2024-09-21T09:54:05.553 INFO:tasks.rados.rados.0.smithi033.stdout:917: done (1 left) 2024-09-21T09:54:05.553 INFO:tasks.rados.rados.0.smithi033.stdout:918: read oid 49 snap -1 2024-09-21T09:54:05.553 INFO:tasks.rados.rados.0.smithi033.stdout:918: expect (ObjNum 256 snap 65 seq_num 256) 2024-09-21T09:54:05.553 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 80 2024-09-21T09:54:05.553 INFO:tasks.rados.rados.0.smithi033.stdout:919: seq_num 306 ranges {4825088=327680} 2024-09-21T09:54:05.582 INFO:tasks.rados.rados.0.smithi033.stdout:919: writing smithi03385152-28 from 4825088 to 5152768 tid 1 2024-09-21T09:54:05.582 INFO:tasks.rados.rados.0.smithi033.stdout:920: read oid 39 snap -1 2024-09-21T09:54:05.582 INFO:tasks.rados.rados.0.smithi033.stdout:920: expect deleted 2024-09-21T09:54:05.582 INFO:tasks.rados.rados.0.smithi033.stdout:921: rmattr oid 12 current snap is 80 2024-09-21T09:54:05.582 INFO:tasks.rados.rados.0.smithi033.stdout:921: done (4 left) 2024-09-21T09:54:05.582 INFO:tasks.rados.rados.0.smithi033.stdout:922: delete oid 3 current snap is 80 2024-09-21T09:54:05.583 INFO:tasks.rados.rados.0.smithi033.stdout:916: finishing write tid 1 to smithi03385152-21 2024-09-21T09:54:05.583 INFO:tasks.rados.rados.0.smithi033.stdout:916: finishing write tid 2 to smithi03385152-21 2024-09-21T09:54:05.583 INFO:tasks.rados.rados.0.smithi033.stdout:916: finishing write tid 3 to smithi03385152-21 2024-09-21T09:54:05.583 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 353 (ObjNum 304 snap 80 seq_num 304) dirty exists 2024-09-21T09:54:05.583 INFO:tasks.rados.rados.0.smithi033.stdout:916: left oid 21 (ObjNum 304 snap 80 seq_num 304) 2024-09-21T09:54:05.583 INFO:tasks.rados.rados.0.smithi033.stdout:918: expect (ObjNum 256 snap 65 seq_num 256) 2024-09-21T09:54:05.632 INFO:tasks.rados.rados.0.smithi033.stdout:919: finishing write tid 1 to smithi03385152-28 2024-09-21T09:54:05.632 INFO:tasks.rados.rados.0.smithi033.stdout:919: finishing write tid 2 to smithi03385152-28 2024-09-21T09:54:05.632 INFO:tasks.rados.rados.0.smithi033.stdout:919: finishing write tid 3 to smithi03385152-28 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 147 (ObjNum 306 snap 80 seq_num 306) dirty exists 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:919: left oid 28 (ObjNum 306 snap 80 seq_num 306) 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:916: done (4 left) 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:918: done (3 left) 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:919: done (2 left) 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:920: done (1 left) 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:922: done (0 left) 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:923: copy_from oid 50 from oid 30 current snap is 80 2024-09-21T09:54:05.633 INFO:tasks.rados.rados.0.smithi033.stdout:924: snap_remove snap 73 2024-09-21T09:54:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[39045]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:54:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[39045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:05.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[27969]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:54:05.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:05.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:05.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:05.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[27969]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:05.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:05 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:05 smithi105 ceph-mon[31286]: osdmap e229: 8 total, 8 up, 8 in 2024-09-21T09:54:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:05 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:05 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:05 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:05 smithi105 ceph-mon[31286]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-21T09:54:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:05 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:06.338 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:54:06.534 INFO:tasks.rados.rados.0.smithi033.stdout:924: done (1 left) 2024-09-21T09:54:06.534 INFO:tasks.rados.rados.0.smithi033.stdout:925: snap_create 2024-09-21T09:54:06.534 INFO:tasks.rados.rados.0.smithi033.stdout:923: finishing copy_from racing read to smithi03385152-50 2024-09-21T09:54:06.535 INFO:tasks.rados.rados.0.smithi033.stdout:923: finishing copy_from to smithi03385152-50 2024-09-21T09:54:06.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 602 (ObjNum 289 snap 70 seq_num 289) dirty exists 2024-09-21T09:54:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[39045]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:54:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[39045]: pgmap v27: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:54:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:54:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:54:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:54:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:06.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[27969]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:54:06.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:06.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[27969]: pgmap v27: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:54:06.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:54:06.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:54:06.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:54:06.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:06 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:06.898 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:06 smithi105 ceph-mon[31286]: osdmap e230: 8 total, 8 up, 8 in 2024-09-21T09:54:06.898 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:06 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:06.898 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:06 smithi105 ceph-mon[31286]: pgmap v27: 121 pgs: 121 active+clean; 138 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:54:06.898 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:06 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:54:06.898 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:06 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-21T09:54:06.898 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:06 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-21T09:54:06.898 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:06 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:06.920 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:54:06.920 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (58s) 11s ago 5m 15.5M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:54:06.920 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (35s) 11s ago 5m 80.6M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (6m) 11s ago 6m 14.8M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (6m) 11s ago 6m 19.0M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283 running (99s) 11s ago 10m 132M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b10e03ec7c52 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (25s) 11s ago 12m 490M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (12m) 11s ago 12m 58.0M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 220cb0bc67bb 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (10m) 11s ago 10m 49.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c2741af3f853 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (10m) 11s ago 10m 49.3M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f8b68f40cc5 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (53s) 11s ago 4m 7578k - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (30s) 11s ago 4m 7968k - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (10m) 11s ago 10m 136M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (9m) 11s ago 9m 138M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (9m) 11s ago 9m 123M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (8m) 11s ago 8m 83.3M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:54:06.921 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (8m) 11s ago 8m 140M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:54:06.922 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (8m) 11s ago 8m 149M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:54:06.922 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (7m) 11s ago 7m 96.8M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:54:06.922 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (7m) 11s ago 7m 123M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:54:06.922 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 starting - - - - 2024-09-21T09:54:07.269 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:54:07.269 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:54:07.269 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 13, 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:54:07.270 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:54:07.271 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:54:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:07.533 INFO:tasks.rados.rados.0.smithi033.stdout:923: done (1 left) 2024-09-21T09:54:07.533 INFO:tasks.rados.rados.0.smithi033.stdout:925: done (0 left) 2024-09-21T09:54:07.533 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 81 2024-09-21T09:54:07.533 INFO:tasks.rados.rados.0.smithi033.stdout:926: seq_num 308 ranges {4186112=360448} 2024-09-21T09:54:07.558 INFO:tasks.rados.rados.0.smithi033.stdout:926: writing smithi03385152-21 from 4186112 to 4546560 tid 1 2024-09-21T09:54:07.558 INFO:tasks.rados.rados.0.smithi033.stdout:927: rollback oid 42 current snap is 81 2024-09-21T09:54:07.558 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 42 to 74 2024-09-21T09:54:07.558 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 81 2024-09-21T09:54:07.558 INFO:tasks.rados.rados.0.smithi033.stdout:928: seq_num 309 ranges {0=376832} 2024-09-21T09:54:07.561 INFO:tasks.rados.rados.0.smithi033.stdout:928: writing smithi03385152-47 from 0 to 376832 tid 1 2024-09-21T09:54:07.561 INFO:tasks.rados.rados.0.smithi033.stdout:929: copy_from oid 13 from oid 7 current snap is 81 2024-09-21T09:54:07.561 INFO:tasks.rados.rados.0.smithi033.stdout:930: snap_create 2024-09-21T09:54:07.562 INFO:tasks.rados.rados.0.smithi033.stdout:927: finishing rollback tid 0 to smithi03385152-42 2024-09-21T09:54:07.564 INFO:tasks.rados.rados.0.smithi033.stdout:929: finishing copy_from to smithi03385152-13 2024-09-21T09:54:07.564 INFO:tasks.rados.rados.0.smithi033.stdout:929: got expected ENOENT (src dne) 2024-09-21T09:54:07.564 INFO:tasks.rados.rados.0.smithi033.stdout:927: finishing rollback tid 1 to smithi03385152-42 2024-09-21T09:54:07.564 INFO:tasks.rados.rados.0.smithi033.stdout:929: finishing copy_from racing read to smithi03385152-13 2024-09-21T09:54:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:927: finishing rollback tid 2 to smithi03385152-42 2024-09-21T09:54:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 608 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-21T09:54:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:928: finishing write tid 1 to smithi03385152-47 2024-09-21T09:54:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:928: finishing write tid 2 to smithi03385152-47 2024-09-21T09:54:07.566 INFO:tasks.rados.rados.0.smithi033.stdout:928: finishing write tid 3 to smithi03385152-47 2024-09-21T09:54:07.566 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 83 (ObjNum 309 snap 81 seq_num 309) dirty exists 2024-09-21T09:54:07.567 INFO:tasks.rados.rados.0.smithi033.stdout:928: left oid 47 (ObjNum 309 snap 81 seq_num 309) 2024-09-21T09:54:07.567 INFO:tasks.rados.rados.0.smithi033.stdout:926: finishing write tid 1 to smithi03385152-21 2024-09-21T09:54:07.567 INFO:tasks.rados.rados.0.smithi033.stdout:926: finishing write tid 2 to smithi03385152-21 2024-09-21T09:54:07.568 INFO:tasks.rados.rados.0.smithi033.stdout:926: finishing write tid 3 to smithi03385152-21 2024-09-21T09:54:07.568 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 355 (ObjNum 308 snap 81 seq_num 308) dirty exists 2024-09-21T09:54:07.568 INFO:tasks.rados.rados.0.smithi033.stdout:926: left oid 21 (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "2/20 daemons upgraded", 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading mgr daemons", 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:54:07.573 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:54:07.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:07 smithi105 ceph-mon[31286]: Upgrade: Updating mgr.x 2024-09-21T09:54:07.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:07 smithi105 ceph-mon[31286]: Deploying daemon mgr.x on smithi105 2024-09-21T09:54:07.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:07 smithi105 ceph-mon[31286]: from='client.25118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:07 smithi105 ceph-mon[31286]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:54:07.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:07 smithi105 ceph-mon[31286]: from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:07 smithi105 ceph-mon[31286]: from='client.25130 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.669 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:07 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/876753334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[39045]: Upgrade: Updating mgr.x 2024-09-21T09:54:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[39045]: Deploying daemon mgr.x on smithi105 2024-09-21T09:54:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[39045]: from='client.25118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[39045]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:54:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[39045]: from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[39045]: from='client.25130 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/876753334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[27969]: Upgrade: Updating mgr.x 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[27969]: Deploying daemon mgr.x on smithi105 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[27969]: from='client.25118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[27969]: osdmap e231: 8 total, 8 up, 8 in 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[27969]: from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[27969]: from='client.25130 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:07 smithi033 ceph-mon[27969]: from='client.? 172.21.15.33:0/876753334' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:08.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:07 smithi105 systemd[1]: Stopping Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:08.546 INFO:tasks.rados.rados.0.smithi033.stdout:926: done (4 left) 2024-09-21T09:54:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:927: done (3 left) 2024-09-21T09:54:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:928: done (2 left) 2024-09-21T09:54:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:929: done (1 left) 2024-09-21T09:54:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:930: done (0 left) 2024-09-21T09:54:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 82 2024-09-21T09:54:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:931: seq_num 310 ranges {3219456=401408} 2024-09-21T09:54:08.568 INFO:tasks.rados.rados.0.smithi033.stdout:931: writing smithi03385152-15 from 3219456 to 3620864 tid 1 2024-09-21T09:54:08.568 INFO:tasks.rados.rados.0.smithi033.stdout:932: delete oid 31 current snap is 82 2024-09-21T09:54:08.571 INFO:tasks.rados.rados.0.smithi033.stdout:932: done (1 left) 2024-09-21T09:54:08.571 INFO:tasks.rados.rados.0.smithi033.stdout:933: snap_create 2024-09-21T09:54:08.576 INFO:tasks.rados.rados.0.smithi033.stdout:931: finishing write tid 1 to smithi03385152-15 2024-09-21T09:54:08.576 INFO:tasks.rados.rados.0.smithi033.stdout:931: finishing write tid 2 to smithi03385152-15 2024-09-21T09:54:08.577 INFO:tasks.rados.rados.0.smithi033.stdout:931: finishing write tid 3 to smithi03385152-15 2024-09-21T09:54:08.578 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 229 (ObjNum 310 snap 82 seq_num 310) dirty exists 2024-09-21T09:54:08.578 INFO:tasks.rados.rados.0.smithi033.stdout:931: left oid 15 (ObjNum 310 snap 82 seq_num 310) 2024-09-21T09:54:08.613 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:08 smithi105 podman[79088]: 2024-09-21 09:54:08.337789398 +0000 UTC m=+0.219229987 container died b10e03ec7c52c92d932372e2f7be47f83bc400a43de6182794c71ef4fb36cd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T09:54:08.614 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:08 smithi105 ceph-mon[31286]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:54:08.614 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:08 smithi105 ceph-mon[31286]: from='client.15234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:08.614 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:08 smithi105 ceph-mon[31286]: pgmap v30: 121 pgs: 121 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:54:08.991 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:08 smithi105 podman[79088]: 2024-09-21 09:54:08.721499815 +0000 UTC m=+0.602940405 container cleanup b10e03ec7c52c92d932372e2f7be47f83bc400a43de6182794c71ef4fb36cd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:54:08.991 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:08 smithi105 bash[79088]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x 2024-09-21T09:54:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:08 smithi033 ceph-mon[39045]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:54:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:08 smithi033 ceph-mon[39045]: from='client.15234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:08 smithi033 ceph-mon[39045]: pgmap v30: 121 pgs: 121 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:54:08.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:08 smithi033 ceph-mon[27969]: osdmap e232: 8 total, 8 up, 8 in 2024-09-21T09:54:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:08 smithi033 ceph-mon[27969]: from='client.15234 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:08 smithi033 ceph-mon[27969]: pgmap v30: 121 pgs: 121 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:54:09.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:08 smithi105 podman[79105]: 2024-09-21 09:54:08.989401391 +0000 UTC m=+0.651812680 container remove b10e03ec7c52c92d932372e2f7be47f83bc400a43de6182794c71ef4fb36cd54 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T09:54:09.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:08 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-21T09:54:09.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x.service: Failed with result 'exit-code'. 2024-09-21T09:54:09.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 systemd[1]: Stopped Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:09.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x.service: Consumed 18.382s CPU time. 2024-09-21T09:54:09.548 INFO:tasks.rados.rados.0.smithi033.stdout:931: done (1 left) 2024-09-21T09:54:09.548 INFO:tasks.rados.rados.0.smithi033.stdout:933: done (0 left) 2024-09-21T09:54:09.549 INFO:tasks.rados.rados.0.smithi033.stdout:934: copy_from oid 33 from oid 44 current snap is 83 2024-09-21T09:54:09.549 INFO:tasks.rados.rados.0.smithi033.stdout:935: delete oid 3 current snap is 83 2024-09-21T09:54:09.554 INFO:tasks.rados.rados.0.smithi033.stdout:935: done (1 left) 2024-09-21T09:54:09.554 INFO:tasks.rados.rados.0.smithi033.stdout:936: snap_remove snap 80 2024-09-21T09:54:09.694 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:09 smithi105 ceph-mon[31286]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:54:09.695 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 systemd[1]: Starting Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:09.695 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 podman[79241]: 2024-09-21 09:54:09.457068067 +0000 UTC m=+0.016029767 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:54:09.695 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 podman[79241]: 2024-09-21 09:54:09.565636353 +0000 UTC m=+0.124598047 container create 5f55827751a5fb0133b4055ce05bbcc93689fd30fdd4599205c388cda652491e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:54:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:09 smithi033 ceph-mon[39045]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:54:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:09 smithi033 ceph-mon[27969]: osdmap e233: 8 total, 8 up, 8 in 2024-09-21T09:54:10.214 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 podman[79241]: 2024-09-21 09:54:09.801666142 +0000 UTC m=+0.360627843 container init 5f55827751a5fb0133b4055ce05bbcc93689fd30fdd4599205c388cda652491e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:54:10.214 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 podman[79241]: 2024-09-21 09:54:09.806504375 +0000 UTC m=+0.365466071 container start 5f55827751a5fb0133b4055ce05bbcc93689fd30fdd4599205c388cda652491e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T09:54:10.214 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 ceph-mgr[79279]: -- 172.21.15.105:0/2430981233 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56025a2770e0 con 0x56025a255000 2024-09-21T09:54:10.214 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 bash[79241]: 5f55827751a5fb0133b4055ce05bbcc93689fd30fdd4599205c388cda652491e 2024-09-21T09:54:10.214 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:09 smithi105 systemd[1]: Started Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:10.563 INFO:tasks.rados.rados.0.smithi033.stdout:934: finishing copy_from to smithi03385152-33 2024-09-21T09:54:10.565 INFO:tasks.rados.rados.0.smithi033.stdout:934: got expected ENOENT (src dne) 2024-09-21T09:54:10.567 INFO:tasks.rados.rados.0.smithi033.stdout:934: finishing copy_from racing read to smithi03385152-33 2024-09-21T09:54:10.567 INFO:tasks.rados.rados.0.smithi033.stdout:934: done (1 left) 2024-09-21T09:54:10.568 INFO:tasks.rados.rados.0.smithi033.stdout:936: done (0 left) 2024-09-21T09:54:10.568 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 83 2024-09-21T09:54:10.568 INFO:tasks.rados.rados.0.smithi033.stdout:937: seq_num 313 ranges {0=450560} 2024-09-21T09:54:10.570 INFO:tasks.rados.rados.0.smithi033.stdout:937: writing smithi03385152-34 from 0 to 450560 tid 1 2024-09-21T09:54:10.570 INFO:tasks.rados.rados.0.smithi033.stdout:938: read oid 25 snap -1 2024-09-21T09:54:10.571 INFO:tasks.rados.rados.0.smithi033.stdout:938: expect (ObjNum 243 snap 63 seq_num 243) 2024-09-21T09:54:10.571 INFO:tasks.rados.rados.0.smithi033.stdout:939: read oid 39 snap -1 2024-09-21T09:54:10.571 INFO:tasks.rados.rados.0.smithi033.stdout:939: expect deleted 2024-09-21T09:54:10.571 INFO:tasks.rados.rados.0.smithi033.stdout:940: delete oid 37 current snap is 83 2024-09-21T09:54:10.574 INFO:tasks.rados.rados.0.smithi033.stdout:937: finishing write tid 1 to smithi03385152-34 2024-09-21T09:54:10.574 INFO:tasks.rados.rados.0.smithi033.stdout:937: finishing write tid 2 to smithi03385152-34 2024-09-21T09:54:10.575 INFO:tasks.rados.rados.0.smithi033.stdout:939: done (3 left) 2024-09-21T09:54:10.575 INFO:tasks.rados.rados.0.smithi033.stdout:940: done (2 left) 2024-09-21T09:54:10.575 INFO:tasks.rados.rados.0.smithi033.stdout:941: read oid 15 snap -1 2024-09-21T09:54:10.575 INFO:tasks.rados.rados.0.smithi033.stdout:941: expect (ObjNum 310 snap 82 seq_num 310) 2024-09-21T09:54:10.575 INFO:tasks.rados.rados.0.smithi033.stdout:942: delete oid 19 current snap is 83 2024-09-21T09:54:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:937: finishing write tid 3 to smithi03385152-34 2024-09-21T09:54:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 601 (ObjNum 313 snap 83 seq_num 313) dirty exists 2024-09-21T09:54:10.576 INFO:tasks.rados.rados.0.smithi033.stdout:937: left oid 34 (ObjNum 313 snap 83 seq_num 313) 2024-09-21T09:54:10.578 INFO:tasks.rados.rados.0.smithi033.stdout:937: done (3 left) 2024-09-21T09:54:10.579 INFO:tasks.rados.rados.0.smithi033.stdout:942: done (2 left) 2024-09-21T09:54:10.579 INFO:tasks.rados.rados.0.smithi033.stdout:943: read oid 47 snap -1 2024-09-21T09:54:10.579 INFO:tasks.rados.rados.0.smithi033.stdout:943: expect (ObjNum 309 snap 81 seq_num 309) 2024-09-21T09:54:10.579 INFO:tasks.rados.rados.0.smithi033.stdout:944: snap_remove snap 79 2024-09-21T09:54:10.857 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:10 smithi105 ceph-mon[31286]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:54:10.857 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:10 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:10.857 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:10 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:10.857 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:10 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:10.857 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:10 smithi105 ceph-mon[31286]: pgmap v33: 121 pgs: 121 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:54:10.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[39045]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:54:10.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:10.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:10.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:10.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[39045]: pgmap v33: 121 pgs: 121 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:54:10.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[27969]: osdmap e234: 8 total, 8 up, 8 in 2024-09-21T09:54:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:10.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:10 smithi033 ceph-mon[27969]: pgmap v33: 121 pgs: 121 active+clean; 143 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 1.2 MiB/s wr, 20 op/s 2024-09-21T09:54:11.506 INFO:tasks.rados.rados.0.smithi033.stdout:944: done (3 left) 2024-09-21T09:54:11.506 INFO:tasks.rados.rados.0.smithi033.stdout:945: snap_remove snap 82 2024-09-21T09:54:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:11 smithi033 ceph-mon[39045]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:54:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:11 smithi033 ceph-mon[39045]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:54:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:11 smithi033 ceph-mon[27969]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:54:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:11 smithi033 ceph-mon[27969]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:54:11.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:54:11] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.51.0" 2024-09-21T09:54:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:11 smithi105 ceph-mon[31286]: osdmap e235: 8 total, 8 up, 8 in 2024-09-21T09:54:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:11 smithi105 ceph-mon[31286]: osdmap e236: 8 total, 8 up, 8 in 2024-09-21T09:54:12.502 INFO:tasks.rados.rados.0.smithi033.stdout:945: done (3 left) 2024-09-21T09:54:12.503 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 83 2024-09-21T09:54:12.503 INFO:tasks.rados.rados.0.smithi033.stdout:946: seq_num 316 ranges {0=499712} 2024-09-21T09:54:12.506 INFO:tasks.rados.rados.0.smithi033.stdout:946: writing smithi03385152-37 from 0 to 499712 tid 1 2024-09-21T09:54:12.506 INFO:tasks.rados.rados.0.smithi033.stdout:947: setattr oid 8 current snap is 83 2024-09-21T09:54:12.506 INFO:tasks.rados.rados.0.smithi033.stdout:943: expect (ObjNum 309 snap 81 seq_num 309) 2024-09-21T09:54:12.518 INFO:tasks.rados.rados.0.smithi033.stdout:938: expect (ObjNum 243 snap 63 seq_num 243) 2024-09-21T09:54:12.714 INFO:tasks.rados.rados.0.smithi033.stdout:941: expect (ObjNum 310 snap 82 seq_num 310) 2024-09-21T09:54:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:12 smithi105 ceph-mon[31286]: pgmap v36: 121 pgs: 121 active+clean; 144 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 388 KiB/s wr, 7 op/s 2024-09-21T09:54:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:12 smithi105 ceph-mon[31286]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:54:12.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:12 smithi033 ceph-mon[39045]: pgmap v36: 121 pgs: 121 active+clean; 144 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 388 KiB/s wr, 7 op/s 2024-09-21T09:54:12.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:12 smithi033 ceph-mon[39045]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:54:12.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:12 smithi033 ceph-mon[27969]: pgmap v36: 121 pgs: 121 active+clean; 144 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 388 KiB/s wr, 7 op/s 2024-09-21T09:54:12.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:12 smithi033 ceph-mon[27969]: osdmap e237: 8 total, 8 up, 8 in 2024-09-21T09:54:12.935 INFO:tasks.rados.rados.0.smithi033.stdout:946: finishing write tid 1 to smithi03385152-37 2024-09-21T09:54:12.935 INFO:tasks.rados.rados.0.smithi033.stdout:946: finishing write tid 2 to smithi03385152-37 2024-09-21T09:54:12.935 INFO:tasks.rados.rados.0.smithi033.stdout:946: finishing write tid 3 to smithi03385152-37 2024-09-21T09:54:12.936 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 370 (ObjNum 316 snap 83 seq_num 316) dirty exists 2024-09-21T09:54:12.936 INFO:tasks.rados.rados.0.smithi033.stdout:946: left oid 37 (ObjNum 316 snap 83 seq_num 316) 2024-09-21T09:54:12.936 INFO:tasks.rados.rados.0.smithi033.stdout:938: done (4 left) 2024-09-21T09:54:12.936 INFO:tasks.rados.rados.0.smithi033.stdout:941: done (3 left) 2024-09-21T09:54:12.936 INFO:tasks.rados.rados.0.smithi033.stdout:943: done (2 left) 2024-09-21T09:54:12.936 INFO:tasks.rados.rados.0.smithi033.stdout:946: done (1 left) 2024-09-21T09:54:12.936 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 83 2024-09-21T09:54:12.936 INFO:tasks.rados.rados.0.smithi033.stdout:948: seq_num 317 ranges {3211264=516096} 2024-09-21T09:54:12.952 INFO:tasks.rados.rados.0.smithi033.stdout:948: writing smithi03385152-23 from 3211264 to 3727360 tid 1 2024-09-21T09:54:12.952 INFO:tasks.rados.rados.0.smithi033.stdout:949: rollback oid 39 current snap is 83 2024-09-21T09:54:12.952 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 39 to 75 2024-09-21T09:54:12.952 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 235 (ObjNum 273 snap 69 seq_num 273) dirty exists 2024-09-21T09:54:12.952 INFO:tasks.rados.rados.0.smithi033.stdout:947: done (2 left) 2024-09-21T09:54:12.952 INFO:tasks.rados.rados.0.smithi033.stdout:950: snap_remove snap 74 2024-09-21T09:54:13.575 INFO:tasks.rados.rados.0.smithi033.stdout:950: done (2 left) 2024-09-21T09:54:13.575 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 83 2024-09-21T09:54:13.575 INFO:tasks.rados.rados.0.smithi033.stdout:951: seq_num 318 ranges {4120576=532480} 2024-09-21T09:54:13.601 INFO:tasks.rados.rados.0.smithi033.stdout:951: writing smithi03385152-17 from 4120576 to 4653056 tid 1 2024-09-21T09:54:13.601 INFO:tasks.rados.rados.0.smithi033.stdout:952: read oid 2 snap -1 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:952: expect (ObjNum 297 snap 76 seq_num 297) 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:953: setattr oid 29 current snap is 83 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:949: finishing rollback tid 1 to smithi03385152-39 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:949: finishing rollback tid 2 to smithi03385152-39 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 101 (ObjNum 199 snap 47 seq_num 199) dirty exists 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:948: finishing write tid 1 to smithi03385152-23 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:948: finishing write tid 2 to smithi03385152-23 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:948: finishing write tid 3 to smithi03385152-23 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 357 (ObjNum 317 snap 83 seq_num 317) dirty exists 2024-09-21T09:54:13.602 INFO:tasks.rados.rados.0.smithi033.stdout:948: left oid 23 (ObjNum 317 snap 83 seq_num 317) 2024-09-21T09:54:13.603 INFO:tasks.rados.rados.0.smithi033.stdout:948: done (4 left) 2024-09-21T09:54:13.603 INFO:tasks.rados.rados.0.smithi033.stdout:949: done (3 left) 2024-09-21T09:54:13.603 INFO:tasks.rados.rados.0.smithi033.stdout:954: delete oid 12 current snap is 83 2024-09-21T09:54:13.605 INFO:tasks.rados.rados.0.smithi033.stdout:951: finishing write tid 1 to smithi03385152-17 2024-09-21T09:54:13.605 INFO:tasks.rados.rados.0.smithi033.stdout:951: finishing write tid 2 to smithi03385152-17 2024-09-21T09:54:13.607 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 625 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T09:54:13.607 INFO:tasks.rados.rados.0.smithi033.stdout:953: done (3 left) 2024-09-21T09:54:13.607 INFO:tasks.rados.rados.0.smithi033.stdout:954: done (2 left) 2024-09-21T09:54:13.607 INFO:tasks.rados.rados.0.smithi033.stdout:955: rollback oid 41 current snap is 83 2024-09-21T09:54:13.607 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 81 2024-09-21T09:54:13.607 INFO:tasks.rados.rados.0.smithi033.stdout:956: rollback oid 23 current snap is 83 2024-09-21T09:54:13.607 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 23 to 78 2024-09-21T09:54:13.608 INFO:tasks.rados.rados.0.smithi033.stdout:957: rmattr oid 32 current snap is 83 2024-09-21T09:54:13.608 INFO:tasks.rados.rados.0.smithi033.stdout:958: read oid 18 snap -1 2024-09-21T09:54:13.608 INFO:tasks.rados.rados.0.smithi033.stdout:958: expect (ObjNum 138 snap 32 seq_num 138) 2024-09-21T09:54:13.608 INFO:tasks.rados.rados.0.smithi033.stdout:959: snap_create 2024-09-21T09:54:13.608 INFO:tasks.rados.rados.0.smithi033.stdout:951: finishing write tid 3 to smithi03385152-17 2024-09-21T09:54:13.608 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 387 (ObjNum 318 snap 83 seq_num 318) dirty exists 2024-09-21T09:54:13.608 INFO:tasks.rados.rados.0.smithi033.stdout:951: left oid 17 (ObjNum 318 snap 83 seq_num 318) 2024-09-21T09:54:13.608 INFO:tasks.rados.rados.0.smithi033.stdout:952: expect (ObjNum 297 snap 76 seq_num 297) 2024-09-21T09:54:13.678 INFO:tasks.rados.rados.0.smithi033.stdout:956: finishing rollback tid 0 to smithi03385152-23 2024-09-21T09:54:13.679 INFO:tasks.rados.rados.0.smithi033.stdout:955: finishing rollback tid 0 to smithi03385152-41 2024-09-21T09:54:13.679 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 105 (ObjNum 295 snap 72 seq_num 295) dirty exists 2024-09-21T09:54:13.679 INFO:tasks.rados.rados.0.smithi033.stdout:956: finishing rollback tid 1 to smithi03385152-23 2024-09-21T09:54:13.679 INFO:tasks.rados.rados.0.smithi033.stdout:955: finishing rollback tid 1 to smithi03385152-41 2024-09-21T09:54:13.679 INFO:tasks.rados.rados.0.smithi033.stdout:956: finishing rollback tid 2 to smithi03385152-23 2024-09-21T09:54:13.679 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 360 (ObjNum 242 snap 63 seq_num 242) dirty exists 2024-09-21T09:54:13.680 INFO:tasks.rados.rados.0.smithi033.stdout:955: finishing rollback tid 2 to smithi03385152-41 2024-09-21T09:54:13.680 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 611 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:54:13.680 INFO:tasks.rados.rados.0.smithi033.stdout:958: expect (ObjNum 138 snap 32 seq_num 138) 2024-09-21T09:54:14.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:14 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:14.576 INFO:tasks.rados.rados.0.smithi033.stdout:951: done (6 left) 2024-09-21T09:54:14.576 INFO:tasks.rados.rados.0.smithi033.stdout:952: done (5 left) 2024-09-21T09:54:14.576 INFO:tasks.rados.rados.0.smithi033.stdout:955: done (4 left) 2024-09-21T09:54:14.576 INFO:tasks.rados.rados.0.smithi033.stdout:956: done (3 left) 2024-09-21T09:54:14.576 INFO:tasks.rados.rados.0.smithi033.stdout:957: done (2 left) 2024-09-21T09:54:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:958: done (1 left) 2024-09-21T09:54:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:959: done (0 left) 2024-09-21T09:54:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:960: snap_remove snap 81 2024-09-21T09:54:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:14 smithi033 ceph-mon[27969]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:54:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:14 smithi033 ceph-mon[27969]: pgmap v39: 121 pgs: 121 active+clean; 144 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 388 KiB/s wr, 7 op/s 2024-09-21T09:54:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:14 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:14 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:14 smithi033 ceph-mon[39045]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:54:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:14 smithi033 ceph-mon[39045]: pgmap v39: 121 pgs: 121 active+clean; 144 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 388 KiB/s wr, 7 op/s 2024-09-21T09:54:14.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:14 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:14.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:14 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:14 smithi105 ceph-mon[31286]: osdmap e238: 8 total, 8 up, 8 in 2024-09-21T09:54:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:14 smithi105 ceph-mon[31286]: pgmap v39: 121 pgs: 121 active+clean; 144 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 388 KiB/s wr, 7 op/s 2024-09-21T09:54:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:14 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:14 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.422 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: [21/Sep/2024:09:54:15] ENGINE Bus STARTING 2024-09-21T09:54:15.422 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: CherryPy Checker: 2024-09-21T09:54:15.422 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: The Application mounted at '' has an empty config. 2024-09-21T09:54:15.422 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: 2024-09-21T09:54:15.423 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: [21/Sep/2024:09:54:15] ENGINE Serving on http://:::9283 2024-09-21T09:54:15.423 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: [21/Sep/2024:09:54:15] ENGINE Bus STARTED 2024-09-21T09:54:15.583 INFO:tasks.rados.rados.0.smithi033.stdout:960: done (0 left) 2024-09-21T09:54:15.583 INFO:tasks.rados.rados.0.smithi033.stdout:961: read oid 47 snap -1 2024-09-21T09:54:15.583 INFO:tasks.rados.rados.0.smithi033.stdout:961: expect (ObjNum 309 snap 81 seq_num 309) 2024-09-21T09:54:15.583 INFO:tasks.rados.rados.0.smithi033.stdout:962: read oid 35 snap -1 2024-09-21T09:54:15.584 INFO:tasks.rados.rados.0.smithi033.stdout:962: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:54:15.584 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 84 2024-09-21T09:54:15.584 INFO:tasks.rados.rados.0.smithi033.stdout:963: seq_num 320 ranges {6029312=565248} 2024-09-21T09:54:15.624 INFO:tasks.rados.rados.0.smithi033.stdout:963: writing smithi03385152-4 from 6029312 to 6594560 tid 1 2024-09-21T09:54:15.624 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 84 2024-09-21T09:54:15.624 INFO:tasks.rados.rados.0.smithi033.stdout:964: seq_num 321 ranges {0=581632} 2024-09-21T09:54:15.627 INFO:tasks.rados.rados.0.smithi033.stdout:964: writing smithi03385152-36 from 0 to 581632 tid 1 2024-09-21T09:54:15.627 INFO:tasks.rados.rados.0.smithi033.stdout:965: copy_from oid 8 from oid 27 current snap is 84 2024-09-21T09:54:15.627 INFO:tasks.rados.rados.0.smithi033.stdout:961: expect (ObjNum 309 snap 81 seq_num 309) 2024-09-21T09:54:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:962: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:54:15.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:54:15.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: Standby manager daemon x restarted 2024-09-21T09:54:15.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: Standby manager daemon x started 2024-09-21T09:54:15.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:15.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:15.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:15.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:15.906 INFO:tasks.rados.rados.0.smithi033.stdout:963: finishing write tid 1 to smithi03385152-4 2024-09-21T09:54:15.906 INFO:tasks.rados.rados.0.smithi033.stdout:963: finishing write tid 2 to smithi03385152-4 2024-09-21T09:54:15.906 INFO:tasks.rados.rados.0.smithi033.stdout:961: done (4 left) 2024-09-21T09:54:15.907 INFO:tasks.rados.rados.0.smithi033.stdout:962: done (3 left) 2024-09-21T09:54:15.907 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 84 2024-09-21T09:54:15.907 INFO:tasks.rados.rados.0.smithi033.stdout:964: finishing write tid 1 to smithi03385152-36 2024-09-21T09:54:15.907 INFO:tasks.rados.rados.0.smithi033.stdout:966: seq_num 322 ranges {5275648=598016} 2024-09-21T09:54:15.938 INFO:tasks.rados.rados.0.smithi033.stdout:966: writing smithi03385152-18 from 5275648 to 5873664 tid 1 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:967: rollback oid 34 current snap is 84 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 83 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:964: finishing write tid 2 to smithi03385152-36 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:963: finishing write tid 3 to smithi03385152-4 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 165 (ObjNum 320 snap 84 seq_num 320) dirty exists 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:963: left oid 4 (ObjNum 320 snap 84 seq_num 320) 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:964: finishing write tid 3 to smithi03385152-36 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 627 (ObjNum 321 snap 84 seq_num 321) dirty exists 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:964: left oid 36 (ObjNum 321 snap 84 seq_num 321) 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:965: finishing copy_from racing read to smithi03385152-8 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:963: done (4 left) 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:964: done (3 left) 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:968: delete oid 47 current snap is 84 2024-09-21T09:54:15.939 INFO:tasks.rados.rados.0.smithi033.stdout:965: finishing copy_from to smithi03385152-8 2024-09-21T09:54:15.940 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 316 (ObjNum 290 snap 70 seq_num 290) dirty exists 2024-09-21T09:54:15.940 INFO:tasks.rados.rados.0.smithi033.stdout:965: done (3 left) 2024-09-21T09:54:15.940 INFO:tasks.rados.rados.0.smithi033.stdout:968: done (2 left) 2024-09-21T09:54:15.941 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 84 2024-09-21T09:54:15.941 INFO:tasks.rados.rados.0.smithi033.stdout:969: seq_num 324 ranges {0=630784} 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:969: writing smithi03385152-7 from 0 to 630784 tid 1 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:970: read oid 21 snap -1 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:970: expect (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:971: read oid 35 snap -1 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:971: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 84 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:966: finishing write tid 1 to smithi03385152-18 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:966: finishing write tid 2 to smithi03385152-18 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:967: finishing rollback tid 0 to smithi03385152-34 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:967: finishing rollback tid 1 to smithi03385152-34 2024-09-21T09:54:15.945 INFO:tasks.rados.rados.0.smithi033.stdout:972: seq_num 325 ranges {2580480=647168} 2024-09-21T09:54:15.965 INFO:tasks.rados.rados.0.smithi033.stdout:972: writing smithi03385152-27 from 2580480 to 3227648 tid 1 2024-09-21T09:54:15.966 INFO:tasks.rados.rados.0.smithi033.stdout:973: copy_from oid 45 from oid 3 current snap is 84 2024-09-21T09:54:15.966 INFO:tasks.rados.rados.0.smithi033.stdout:967: finishing rollback tid 2 to smithi03385152-34 2024-09-21T09:54:15.966 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 619 (ObjNum 313 snap 83 seq_num 313) dirty exists 2024-09-21T09:54:15.966 INFO:tasks.rados.rados.0.smithi033.stdout:966: finishing write tid 3 to smithi03385152-18 2024-09-21T09:54:15.966 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 124 (ObjNum 322 snap 84 seq_num 322) dirty exists 2024-09-21T09:54:15.966 INFO:tasks.rados.rados.0.smithi033.stdout:966: left oid 18 (ObjNum 322 snap 84 seq_num 322) 2024-09-21T09:54:15.966 INFO:tasks.rados.rados.0.smithi033.stdout:970: expect (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: Standby manager daemon x restarted 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: Standby manager daemon x started 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: osdmap e239: 8 total, 8 up, 8 in 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: Standby manager daemon x restarted 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: Standby manager daemon x started 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:15 smithi033 ceph-mon[27969]: from='mgr.? 172.21.15.105:0/2791371188' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:16.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:15.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:16.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:16.311 INFO:tasks.rados.rados.0.smithi033.stdout:969: finishing write tid 1 to smithi03385152-7 2024-09-21T09:54:16.312 INFO:tasks.rados.rados.0.smithi033.stdout:969: finishing write tid 2 to smithi03385152-7 2024-09-21T09:54:16.312 INFO:tasks.rados.rados.0.smithi033.stdout:971: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:54:16.579 INFO:tasks.rados.rados.0.smithi033.stdout:969: finishing write tid 3 to smithi03385152-7 2024-09-21T09:54:16.579 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 629 (ObjNum 324 snap 84 seq_num 324) dirty exists 2024-09-21T09:54:16.579 INFO:tasks.rados.rados.0.smithi033.stdout:969: left oid 7 (ObjNum 324 snap 84 seq_num 324) 2024-09-21T09:54:16.579 INFO:tasks.rados.rados.0.smithi033.stdout:973: finishing copy_from to smithi03385152-45 2024-09-21T09:54:16.579 INFO:tasks.rados.rados.0.smithi033.stdout:973: got expected ENOENT (src dne) 2024-09-21T09:54:16.579 INFO:tasks.rados.rados.0.smithi033.stdout:973: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:54:16.579 INFO:tasks.rados.rados.0.smithi033.stdout:972: finishing write tid 1 to smithi03385152-27 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:966: done (6 left) 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:967: done (5 left) 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:969: done (4 left) 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:970: done (3 left) 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:971: done (2 left) 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:973: done (1 left) 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:974: read oid 41 snap 83 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:974: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:975: rollback oid 12 current snap is 84 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 12 to 83 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:972: finishing write tid 2 to smithi03385152-27 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:972: finishing write tid 3 to smithi03385152-27 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 325 (ObjNum 325 snap 84 seq_num 325) dirty exists 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:972: left oid 27 (ObjNum 325 snap 84 seq_num 325) 2024-09-21T09:54:16.580 INFO:tasks.rados.rados.0.smithi033.stdout:972: done (2 left) 2024-09-21T09:54:16.581 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 84 2024-09-21T09:54:16.581 INFO:tasks.rados.rados.0.smithi033.stdout:976: seq_num 326 ranges {3620864=663552} 2024-09-21T09:54:16.602 INFO:tasks.rados.rados.0.smithi033.stdout:976: writing smithi03385152-15 from 3620864 to 4284416 tid 1 2024-09-21T09:54:16.602 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 84 2024-09-21T09:54:16.602 INFO:tasks.rados.rados.0.smithi033.stdout:975: finishing rollback tid 1 to smithi03385152-12 2024-09-21T09:54:16.603 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 389 (ObjNum 225 snap 0 seq_num 90620736) dirty dne 2024-09-21T09:54:16.603 INFO:tasks.rados.rados.0.smithi033.stdout:974: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:54:16.755 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:16 smithi105 ceph-mon[31286]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:54:16.755 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:16 smithi105 ceph-mon[31286]: mgrmap e36: y(active, since 25s), standbys: x 2024-09-21T09:54:16.755 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:16 smithi105 ceph-mon[31286]: pgmap v42: 121 pgs: 121 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 138 KiB/s wr, 0 op/s 2024-09-21T09:54:16.869 INFO:tasks.rados.rados.0.smithi033.stdout:976: finishing write tid 1 to smithi03385152-15 2024-09-21T09:54:16.869 INFO:tasks.rados.rados.0.smithi033.stdout:976: finishing write tid 2 to smithi03385152-15 2024-09-21T09:54:16.869 INFO:tasks.rados.rados.0.smithi033.stdout:976: finishing write tid 3 to smithi03385152-15 2024-09-21T09:54:16.869 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 318 (ObjNum 326 snap 84 seq_num 326) dirty exists 2024-09-21T09:54:16.869 INFO:tasks.rados.rados.0.smithi033.stdout:976: left oid 15 (ObjNum 326 snap 84 seq_num 326) 2024-09-21T09:54:16.869 INFO:tasks.rados.rados.0.smithi033.stdout:977: seq_num 327 ranges {245760=679936} 2024-09-21T09:54:16.874 INFO:tasks.rados.rados.0.smithi033.stdout:977: writing smithi03385152-33 from 245760 to 925696 tid 1 2024-09-21T09:54:16.874 INFO:tasks.rados.rados.0.smithi033.stdout:974: done (3 left) 2024-09-21T09:54:16.875 INFO:tasks.rados.rados.0.smithi033.stdout:975: done (2 left) 2024-09-21T09:54:16.875 INFO:tasks.rados.rados.0.smithi033.stdout:976: done (1 left) 2024-09-21T09:54:16.875 INFO:tasks.rados.rados.0.smithi033.stdout:978: snap_remove snap 75 2024-09-21T09:54:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:16 smithi033 ceph-mon[39045]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:54:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:16 smithi033 ceph-mon[39045]: mgrmap e36: y(active, since 25s), standbys: x 2024-09-21T09:54:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:16 smithi033 ceph-mon[39045]: pgmap v42: 121 pgs: 121 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 138 KiB/s wr, 0 op/s 2024-09-21T09:54:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:16 smithi033 ceph-mon[27969]: osdmap e240: 8 total, 8 up, 8 in 2024-09-21T09:54:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:16 smithi033 ceph-mon[27969]: mgrmap e36: y(active, since 25s), standbys: x 2024-09-21T09:54:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:16 smithi033 ceph-mon[27969]: pgmap v42: 121 pgs: 121 active+clean; 145 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 138 KiB/s wr, 0 op/s 2024-09-21T09:54:17.017 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:16 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:17.603 INFO:tasks.rados.rados.0.smithi033.stdout:978: done (1 left) 2024-09-21T09:54:17.603 INFO:tasks.rados.rados.0.smithi033.stdout:979: rmattr oid 18 current snap is 84 2024-09-21T09:54:17.603 INFO:tasks.rados.rados.0.smithi033.stdout:979: done (1 left) 2024-09-21T09:54:17.603 INFO:tasks.rados.rados.0.smithi033.stdout:980: setattr oid 16 current snap is 84 2024-09-21T09:54:17.603 INFO:tasks.rados.rados.0.smithi033.stdout:977: finishing write tid 1 to smithi03385152-33 2024-09-21T09:54:17.604 INFO:tasks.rados.rados.0.smithi033.stdout:977: finishing write tid 2 to smithi03385152-33 2024-09-21T09:54:17.604 INFO:tasks.rados.rados.0.smithi033.stdout:977: finishing write tid 3 to smithi03385152-33 2024-09-21T09:54:17.604 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 328 (ObjNum 327 snap 84 seq_num 327) dirty exists 2024-09-21T09:54:17.604 INFO:tasks.rados.rados.0.smithi033.stdout:977: left oid 33 (ObjNum 327 snap 84 seq_num 327) 2024-09-21T09:54:17.606 INFO:tasks.rados.rados.0.smithi033.stdout:977: done (1 left) 2024-09-21T09:54:17.607 INFO:tasks.rados.rados.0.smithi033.stdout:981: snap_remove snap 78 2024-09-21T09:54:18.600 INFO:tasks.rados.rados.0.smithi033.stdout:981: done (1 left) 2024-09-21T09:54:18.600 INFO:tasks.rados.rados.0.smithi033.stdout:982: rollback oid 17 current snap is 84 2024-09-21T09:54:18.600 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 83 2024-09-21T09:54:18.600 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 631 (ObjNum 250 snap 65 seq_num 250) dirty exists 2024-09-21T09:54:18.600 INFO:tasks.rados.rados.0.smithi033.stdout:980: done (1 left) 2024-09-21T09:54:18.601 INFO:tasks.rados.rados.0.smithi033.stdout:983: rmattr oid 36 current snap is 84 2024-09-21T09:54:18.601 INFO:tasks.rados.rados.0.smithi033.stdout:984: snap_create 2024-09-21T09:54:18.601 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 0 (ObjNum 321 snap 84 seq_num 321) dirty exists 2024-09-21T09:54:18.613 INFO:tasks.rados.rados.0.smithi033.stdout:982: finishing rollback tid 0 to smithi03385152-17 2024-09-21T09:54:18.613 INFO:tasks.rados.rados.0.smithi033.stdout:982: finishing rollback tid 1 to smithi03385152-17 2024-09-21T09:54:18.613 INFO:tasks.rados.rados.0.smithi033.stdout:982: finishing rollback tid 2 to smithi03385152-17 2024-09-21T09:54:18.613 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 404 (ObjNum 318 snap 83 seq_num 318) dirty exists 2024-09-21T09:54:18.920 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: pgmap v44: 121 pgs: 121 active+clean; 153 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:54:18.921 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: pgmap v44: 121 pgs: 121 active+clean; 153 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:54:18.922 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:18.923 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:18 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:54:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: osdmap e241: 8 total, 8 up, 8 in 2024-09-21T09:54:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: pgmap v44: 121 pgs: 121 active+clean; 153 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:18 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-21T09:54:19.599 INFO:tasks.rados.rados.0.smithi033.stdout:982: done (2 left) 2024-09-21T09:54:19.600 INFO:tasks.rados.rados.0.smithi033.stdout:983: done (1 left) 2024-09-21T09:54:19.600 INFO:tasks.rados.rados.0.smithi033.stdout:984: done (0 left) 2024-09-21T09:54:19.600 INFO:tasks.rados.rados.0.smithi033.stdout:985: copy_from oid 33 from oid 24 current snap is 85 2024-09-21T09:54:19.600 INFO:tasks.rados.rados.0.smithi033.stdout:986: snap_remove snap 83 2024-09-21T09:54:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:19 smithi033 ceph-mon[39045]: Upgrade: Setting container_image for all mgr 2024-09-21T09:54:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:19 smithi033 ceph-mon[39045]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:54:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:19 smithi033 ceph-mon[39045]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:54:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:19 smithi033 ceph-mon[39045]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:54:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:19 smithi033 ceph-mon[27969]: Upgrade: Setting container_image for all mgr 2024-09-21T09:54:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:19 smithi033 ceph-mon[27969]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:54:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:19 smithi033 ceph-mon[27969]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:54:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:19 smithi033 ceph-mon[27969]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:54:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:19 smithi105 ceph-mon[31286]: Upgrade: Setting container_image for all mgr 2024-09-21T09:54:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:19 smithi105 ceph-mon[31286]: Upgrade: It appears safe to stop mon.a 2024-09-21T09:54:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:19 smithi105 ceph-mon[31286]: osdmap e242: 8 total, 8 up, 8 in 2024-09-21T09:54:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:19 smithi105 ceph-mon[31286]: osdmap e243: 8 total, 8 up, 8 in 2024-09-21T09:54:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:20.604 INFO:tasks.rados.rados.0.smithi033.stdout:986: done (1 left) 2024-09-21T09:54:20.604 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 85 2024-09-21T09:54:20.604 INFO:tasks.rados.rados.0.smithi033.stdout:987: seq_num 328 ranges {581632=696320} 2024-09-21T09:54:20.611 INFO:tasks.rados.rados.0.smithi033.stdout:987: writing smithi03385152-36 from 581632 to 1277952 tid 1 2024-09-21T09:54:20.611 INFO:tasks.rados.rados.0.smithi033.stdout:988: delete oid 25 current snap is 85 2024-09-21T09:54:20.611 INFO:tasks.rados.rados.0.smithi033.stdout:985: finishing copy_from racing read to smithi03385152-33 2024-09-21T09:54:20.611 INFO:tasks.rados.rados.0.smithi033.stdout:985: finishing copy_from to smithi03385152-33 2024-09-21T09:54:20.611 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 157 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:54:20.613 INFO:tasks.rados.rados.0.smithi033.stdout:985: done (2 left) 2024-09-21T09:54:20.613 INFO:tasks.rados.rados.0.smithi033.stdout:988: done (1 left) 2024-09-21T09:54:20.613 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 85 2024-09-21T09:54:20.614 INFO:tasks.rados.rados.0.smithi033.stdout:989: seq_num 330 ranges {0=737280} 2024-09-21T09:54:20.618 INFO:tasks.rados.rados.0.smithi033.stdout:989: writing smithi03385152-19 from 0 to 737280 tid 1 2024-09-21T09:54:20.618 INFO:tasks.rados.rados.0.smithi033.stdout:990: rollback oid 35 current snap is 85 2024-09-21T09:54:20.618 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 35 to 84 2024-09-21T09:54:20.619 INFO:tasks.rados.rados.0.smithi033.stdout:987: finishing write tid 1 to smithi03385152-36 2024-09-21T09:54:20.619 INFO:tasks.rados.rados.0.smithi033.stdout:987: finishing write tid 2 to smithi03385152-36 2024-09-21T09:54:20.619 INFO:tasks.rados.rados.0.smithi033.stdout:987: finishing write tid 3 to smithi03385152-36 2024-09-21T09:54:20.619 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 638 (ObjNum 328 snap 85 seq_num 328) dirty exists 2024-09-21T09:54:20.619 INFO:tasks.rados.rados.0.smithi033.stdout:987: left oid 36 (ObjNum 328 snap 85 seq_num 328) 2024-09-21T09:54:20.619 INFO:tasks.rados.rados.0.smithi033.stdout:987: done (2 left) 2024-09-21T09:54:20.619 INFO:tasks.rados.rados.0.smithi033.stdout:991: snap_create 2024-09-21T09:54:20.620 INFO:tasks.rados.rados.0.smithi033.stdout:990: finishing rollback tid 0 to smithi03385152-35 2024-09-21T09:54:20.621 INFO:tasks.rados.rados.0.smithi033.stdout:990: finishing rollback tid 1 to smithi03385152-35 2024-09-21T09:54:20.621 INFO:tasks.rados.rados.0.smithi033.stdout:989: finishing write tid 1 to smithi03385152-19 2024-09-21T09:54:20.621 INFO:tasks.rados.rados.0.smithi033.stdout:990: finishing rollback tid 2 to smithi03385152-35 2024-09-21T09:54:20.621 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 642 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:54:20.621 INFO:tasks.rados.rados.0.smithi033.stdout:989: finishing write tid 2 to smithi03385152-19 2024-09-21T09:54:20.624 INFO:tasks.rados.rados.0.smithi033.stdout:989: finishing write tid 3 to smithi03385152-19 2024-09-21T09:54:20.624 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 177 (ObjNum 330 snap 85 seq_num 330) dirty exists 2024-09-21T09:54:20.624 INFO:tasks.rados.rados.0.smithi033.stdout:989: left oid 19 (ObjNum 330 snap 85 seq_num 330) 2024-09-21T09:54:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[39045]: Upgrade: Updating mon.a 2024-09-21T09:54:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:54:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:54:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[39045]: Deploying daemon mon.a on smithi033 2024-09-21T09:54:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[39045]: pgmap v47: 121 pgs: 121 active+clean; 153 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[39045]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[27969]: Upgrade: Updating mon.a 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[27969]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[27969]: Deploying daemon mon.a on smithi033 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[27969]: pgmap v47: 121 pgs: 121 active+clean; 153 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:54:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:20 smithi033 ceph-mon[27969]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:54:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-mon[31286]: Upgrade: Updating mon.a 2024-09-21T09:54:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='mgr.y' 2024-09-21T09:54:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:54:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:54:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:21.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-mon[31286]: Deploying daemon mon.a on smithi033 2024-09-21T09:54:21.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-mon[31286]: pgmap v47: 121 pgs: 121 active+clean; 153 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.5 MiB/s wr, 25 op/s 2024-09-21T09:54:21.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:20 smithi105 ceph-mon[31286]: osdmap e244: 8 total, 8 up, 8 in 2024-09-21T09:54:21.449 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:21 smithi033 systemd[1]: Stopping Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:21.506 INFO:tasks.rados.rados.0.smithi033.stdout:989: done (2 left) 2024-09-21T09:54:21.506 INFO:tasks.rados.rados.0.smithi033.stdout:990: done (1 left) 2024-09-21T09:54:21.506 INFO:tasks.rados.rados.0.smithi033.stdout:991: done (0 left) 2024-09-21T09:54:21.506 INFO:tasks.rados.rados.0.smithi033.stdout:992: rmattr oid 32 current snap is 86 2024-09-21T09:54:21.506 INFO:tasks.rados.rados.0.smithi033.stdout:993: read oid 14 snap -1 2024-09-21T09:54:21.507 INFO:tasks.rados.rados.0.smithi033.stdout:993: expect (ObjNum 276 snap 69 seq_num 276) 2024-09-21T09:54:21.507 INFO:tasks.rados.rados.0.smithi033.stdout:994: read oid 21 snap -1 2024-09-21T09:54:21.507 INFO:tasks.rados.rados.0.smithi033.stdout:994: expect (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:54:21.507 INFO:tasks.rados.rados.0.smithi033.stdout:995: setattr oid 30 current snap is 86 2024-09-21T09:54:21.507 INFO:tasks.rados.rados.0.smithi033.stdout:996: copy_from oid 9 from oid 40 current snap is 86 2024-09-21T09:54:21.507 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 86 2024-09-21T09:54:21.508 INFO:tasks.rados.rados.0.smithi033.stdout:997: seq_num 331 ranges {5873664=753664} 2024-09-21T09:54:21.545 INFO:tasks.rados.rados.0.smithi033.stdout:997: writing smithi03385152-18 from 5873664 to 6627328 tid 1 2024-09-21T09:54:21.545 INFO:tasks.rados.rados.0.smithi033.stdout:998: snap_remove snap 85 2024-09-21T09:54:21.717 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:54:21] "GET /metrics HTTP/1.1" 200 36204 "" "Prometheus/2.51.0" 2024-09-21T09:54:21.717 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:21 smithi033 ceph-mon[39045]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:21.717 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:21 smithi033 ceph-mon[39045]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:54:21.717 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:21 smithi033 ceph-mon[27969]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:21.717 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:21 smithi033 ceph-mon[27969]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:54:21.718 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a[27946]: 2024-09-21T09:54:21.693+0000 7f901879d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:54:21.718 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a[27946]: 2024-09-21T09:54:21.693+0000 7f901879d640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-21T09:54:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:21 smithi033 podman[96378]: 2024-09-21 09:54:21.715248614 +0000 UTC m=+0.106546731 container died 220cb0bc67bb9d33e368b771580239686f554a4c6452b3b2354cc0ef7d2134de (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T09:54:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:21 smithi105 ceph-mon[31286]: from='mgr.15153 172.21.15.33:0/1969968982' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:21 smithi105 ceph-mon[31286]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:54:22.579 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:22 smithi033 podman[96378]: 2024-09-21 09:54:22.292590192 +0000 UTC m=+0.683888374 container cleanup 220cb0bc67bb9d33e368b771580239686f554a4c6452b3b2354cc0ef7d2134de (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:54:22.579 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:22 smithi033 bash[96378]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a 2024-09-21T09:54:22.951 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:22 smithi033 podman[96395]: 2024-09-21 09:54:22.577592698 +0000 UTC m=+0.859114127 container remove 220cb0bc67bb9d33e368b771580239686f554a4c6452b3b2354cc0ef7d2134de (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T09:54:22.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:22 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a.service: Deactivated successfully. 2024-09-21T09:54:22.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:22 smithi033 systemd[1]: Stopped Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:22.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:22 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a.service: Consumed 15.332s CPU time. 2024-09-21T09:54:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:22 smithi033 systemd[1]: Starting Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 podman[96529]: 2024-09-21 09:54:23.145545061 +0000 UTC m=+0.097702628 container create 652442dcd6a4d173d70116d94b6b0d967af77acdda9e22815dc9bedd7eaf564d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-62da54b) 2024-09-21T09:54:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 podman[96529]: 2024-09-21 09:54:23.064671783 +0000 UTC m=+0.016829310 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:54:23.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 podman[96529]: 2024-09-21 09:54:23.364682405 +0000 UTC m=+0.316839939 container init 652442dcd6a4d173d70116d94b6b0d967af77acdda9e22815dc9bedd7eaf564d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 podman[96529]: 2024-09-21 09:54:23.36949239 +0000 UTC m=+0.321649914 container start 652442dcd6a4d173d70116d94b6b0d967af77acdda9e22815dc9bedd7eaf564d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, RELEASE=main-62da54b, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: pidfile_write: ignore empty --pid-file 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: load: jerasure load: lrc 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Git sha 0 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: DB SUMMARY 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: DB Session ID: JKICXCQWQYRDSJ9S9YNS 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:54:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1172 Bytes 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 4908265 ; 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.env: 0x559b89557120 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.info_log: 0x559b8a707700 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:54:23.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.statistics: (nil) 2024-09-21T09:54:23.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.use_fsync: 0 2024-09-21T09:54:23.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:54:23.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:54:23.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:54:23.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:54:23.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.db_log_dir: 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.wal_dir: 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:54:23.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.write_buffer_manager: 0x559b8a70a1e0 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.unordered_write: 0 2024-09-21T09:54:23.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.row_cache: None 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.wal_filter: None 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.wal_compression: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:54:23.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:54:23.753 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_open_files: -1 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:54:23.754 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Compression algorithms supported: 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: kZSTD supported: 0 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:54:23.755 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.merge_operator: 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_filter: None 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559b8a707320) 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_type: 0 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_block_index_type: 0 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_shortening: 1 2024-09-21T09:54:23.756 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: checksum: 4 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: no_block_cache: 0 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache: 0x559b8a6fb610 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_options: 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: capacity : 536870912 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: num_shard_bits : 4 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: strict_capacity_limit : 0 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_cache_compressed: (nil) 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: persistent_cache: (nil) 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_size: 4096 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_size_deviation: 10 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_restart_interval: 16 2024-09-21T09:54:23.757 INFO:journalctl@ceph.mon.a.smithi033.stdout: index_block_restart_interval: 1 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: metadata_block_size: 4096 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: partition_filters: 0 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: use_delta_encoding: 1 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: filter_policy: bloomfilter 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: whole_key_filtering: 1 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: verify_compression: 0 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: format_version: 5 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: enable_index_compression: 1 2024-09-21T09:54:23.758 INFO:journalctl@ceph.mon.a.smithi033.stdout: block_align: 0 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout: max_auto_readahead_size: 262144 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout: prepopulate_block_cache: 0 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression: NoCompression 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.num_levels: 7 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:54:23.759 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:54:23.760 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:54:23.761 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:54:23.762 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:54:23.762 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:54:23.762 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:54:23.762 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:54:23.762 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:54:23.763 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:54:23.764 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.ttl: 2592000 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:54:23.765 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12814, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 5d0733d4-c276-4395-be99-ec77a7e2a0d0 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912463397501, "job": 1, "event": "recovery_started", "wal_files": [31]} 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912463413191, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 4606500, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12808, "largest_seqno": 15179, "table_properties": {"data_size": 4594231, "index_size": 8663, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2565, "raw_key_size": 24687, "raw_average_key_size": 24, "raw_value_size": 4571048, "raw_average_value_size": 4477, "num_data_blocks": 378, "num_entries": 1021, "num_filter_entries": 1021, "num_deletions": 54, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726912463, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "5d0733d4-c276-4395-be99-ec77a7e2a0d0", "db_session_id": "JKICXCQWQYRDSJ9S9YNS", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912463413327, "job": 1, "event": "recovery_finished"} 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x559b8a72ae00 2024-09-21T09:54:23.766 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: DB pointer 0x559b8a82c000 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** DB Stats ** 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:54:23.767 INFO:journalctl@ceph.mon.a.smithi033.stdout: L0 1/0 4.39 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 524.9 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: L6 1/0 7.64 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: Sum 2/0 12.03 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 524.9 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 524.9 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 524.9 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: Flush(GB): cumulative 0.004, interval 0.004 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:54:23.768 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: Cumulative compaction: 0.00 GB write, 238.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: Interval compaction: 0.00 GB write, 238.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: Block cache BinnedLRUCache@0x559b8a6fb610#7 capacity: 512.00 MB usage: 11.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.78 KB,0.000530481%) IndexBlock(1,9.08 KB,0.00173151%) Misc(1,0.00 KB,0%) 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: starting mon.a rank 0 at public addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] at bind addrs [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: mon.a@-1(???) e3 preinit fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: mon.a@-1(???).mds e7 new map 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: mon.a@-1(???).mds e7 print_map 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: e7 2024-09-21T09:54:23.769 INFO:journalctl@ceph.mon.a.smithi033.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: legacy client fscid: 1 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: Filesystem 'cephfs' (1) 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: fs_name cephfs 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: epoch 7 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: created 2024-09-21T09:47:25.959476+0000 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: modified 2024-09-21T09:47:39.447745+0000 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: tableserver 0 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: root 0 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: session_timeout 60 2024-09-21T09:54:23.770 INFO:journalctl@ceph.mon.a.smithi033.stdout: session_autoclose 300 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: max_file_size 1099511627776 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: max_xattr_size 65536 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: required_client_features {} 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: last_failure 0 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: last_failure_osd_epoch 0 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: max_mds 1 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: in 0 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: up {0=24358} 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: failed 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: damaged 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: stopped 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: data_pools [3] 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: metadata_pool 2 2024-09-21T09:54:23.771 INFO:journalctl@ceph.mon.a.smithi033.stdout: inline_data disabled 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: balancer 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: bal_rank_mask -1 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: standby_count_wanted 0 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: [mds.b{0:24358} state up:active seq 5 addr [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: Standby daemons: 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout: [mds.a{-1:24323} state up:standby seq 1 addr [v2:172.21.15.33:6834/125068843,v1:172.21.15.33:6835/125068843] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: mon.a@-1(???).osd e245 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: mon.a@-1(???).osd e245 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: mon.a@-1(???).osd e245 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:54:23.772 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: mon.a@-1(???).osd e245 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:54:23.773 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 ceph-mon[96568]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-21T09:54:23.773 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 bash[96529]: 652442dcd6a4d173d70116d94b6b0d967af77acdda9e22815dc9bedd7eaf564d 2024-09-21T09:54:23.773 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:23 smithi033 systemd[1]: Started Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:24.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:24.905 INFO:tasks.rados.rados.0.smithi033.stdout:998: done (6 left) 2024-09-21T09:54:24.905 INFO:tasks.rados.rados.0.smithi033.stdout:999: setattr oid 34 current snap is 86 2024-09-21T09:54:24.905 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 113 (ObjNum 295 snap 72 seq_num 295) dirty exists 2024-09-21T09:54:24.905 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 644 (ObjNum 289 snap 70 seq_num 289) dirty exists 2024-09-21T09:54:24.905 INFO:tasks.rados.rados.0.smithi033.stdout:994: expect (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:54:25.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ignoring --setuser ceph since I am not root 2024-09-21T09:54:25.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ignoring --setgroup ceph since I am not root 2024-09-21T09:54:25.247 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mgr[88573]: -- 172.21.15.33:0/3569778573 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562420c750e0 con 0x562420c53000 2024-09-21T09:54:25.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: pgmap v50: 121 pgs: 121 active+clean; 130 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.8 MiB/s wr, 10 op/s 2024-09-21T09:54:25.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: mon.a calling monitor election 2024-09-21T09:54:25.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:54:25.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: monmap epoch 3 2024-09-21T09:54:25.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:25.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: last_changed 2024-09-21T09:43:29.505849+0000 2024-09-21T09:54:25.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:54:25.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: min_mon_release 17 (quincy) 2024-09-21T09:54:25.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: election_strategy: 1 2024-09-21T09:54:25.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:54:25.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:54:25.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:54:25.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:54:25.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:54:25.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: mgrmap e36: y(active, since 34s), standbys: x 2024-09-21T09:54:25.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: overall HEALTH_OK 2024-09-21T09:54:25.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: from='mgr.15153 ' entity='' 2024-09-21T09:54:25.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[39045]: mgrmap e37: y(active, since 34s), standbys: x 2024-09-21T09:54:25.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: pgmap v50: 121 pgs: 121 active+clean; 130 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.8 MiB/s wr, 10 op/s 2024-09-21T09:54:25.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: mon.a calling monitor election 2024-09-21T09:54:25.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:54:25.250 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: monmap epoch 3 2024-09-21T09:54:25.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:25.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: last_changed 2024-09-21T09:43:29.505849+0000 2024-09-21T09:54:25.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:54:25.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: min_mon_release 17 (quincy) 2024-09-21T09:54:25.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: election_strategy: 1 2024-09-21T09:54:25.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:54:25.251 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:54:25.252 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:54:25.252 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:54:25.252 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:54:25.252 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: mgrmap e36: y(active, since 34s), standbys: x 2024-09-21T09:54:25.252 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: overall HEALTH_OK 2024-09-21T09:54:25.252 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: from='mgr.15153 ' entity='' 2024-09-21T09:54:25.253 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:24 smithi033 ceph-mon[96568]: mgrmap e37: y(active, since 34s), standbys: x 2024-09-21T09:54:25.254 INFO:tasks.rados.rados.0.smithi033.stdout:996: finishing copy_from racing read to smithi03385152-9 2024-09-21T09:54:25.254 INFO:tasks.rados.rados.0.smithi033.stdout:996: finishing copy_from to smithi03385152-9 2024-09-21T09:54:25.254 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 140 (ObjNum 114 snap 25 seq_num 114) dirty exists 2024-09-21T09:54:25.255 INFO:tasks.rados.rados.0.smithi033.stdout:997: finishing write tid 1 to smithi03385152-18 2024-09-21T09:54:25.255 INFO:tasks.rados.rados.0.smithi033.stdout:997: finishing write tid 2 to smithi03385152-18 2024-09-21T09:54:25.255 INFO:tasks.rados.rados.0.smithi033.stdout:997: finishing write tid 3 to smithi03385152-18 2024-09-21T09:54:25.255 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 126 (ObjNum 331 snap 86 seq_num 331) dirty exists 2024-09-21T09:54:25.255 INFO:tasks.rados.rados.0.smithi033.stdout:997: left oid 18 (ObjNum 331 snap 86 seq_num 331) 2024-09-21T09:54:25.255 INFO:tasks.rados.rados.0.smithi033.stdout:993: expect (ObjNum 276 snap 69 seq_num 276) 2024-09-21T09:54:25.280 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: ignoring --setuser ceph since I am not root 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: ignoring --setgroup ceph since I am not root 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mgr[79279]: -- 172.21.15.105:0/2258531435 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561c7170b0e0 con 0x561c716e8800 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: pgmap v50: 121 pgs: 121 active+clean; 130 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 2.8 MiB/s wr, 10 op/s 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: mon.a calling monitor election 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: monmap epoch 3 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: last_changed 2024-09-21T09:43:29.505849+0000 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: min_mon_release 17 (quincy) 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: election_strategy: 1 2024-09-21T09:54:25.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:54:25.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:54:25.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:54:25.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:54:25.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: osdmap e245: 8 total, 8 up, 8 in 2024-09-21T09:54:25.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: mgrmap e36: y(active, since 34s), standbys: x 2024-09-21T09:54:25.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: overall HEALTH_OK 2024-09-21T09:54:25.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: from='mgr.15153 ' entity='' 2024-09-21T09:54:25.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:24 smithi105 ceph-mon[31286]: mgrmap e37: y(active, since 34s), standbys: x 2024-09-21T09:54:25.814 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 635 (ObjNum 313 snap 83 seq_num 313) dirty exists 2024-09-21T09:54:25.814 INFO:tasks.rados.rados.0.smithi033.stdout:992: done (6 left) 2024-09-21T09:54:25.814 INFO:tasks.rados.rados.0.smithi033.stdout:993: done (5 left) 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:994: done (4 left) 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:995: done (3 left) 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:996: done (2 left) 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:997: done (1 left) 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:999: done (0 left) 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:1000: rollback oid 41 current snap is 86 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 84 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:1001: read oid 47 snap -1 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:1001: expect deleted 2024-09-21T09:54:25.815 INFO:tasks.rados.rados.0.smithi033.stdout:1002: snap_create 2024-09-21T09:54:25.817 INFO:tasks.rados.rados.0.smithi033.stdout:1000: finishing rollback tid 0 to smithi03385152-41 2024-09-21T09:54:25.817 INFO:tasks.rados.rados.0.smithi033.stdout:1000: finishing rollback tid 1 to smithi03385152-41 2024-09-21T09:54:25.817 INFO:tasks.rados.rados.0.smithi033.stdout:1000: finishing rollback tid 2 to smithi03385152-41 2024-09-21T09:54:25.817 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 614 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:54:25.908 INFO:tasks.rados.rados.0.smithi033.stdout:1000: done (2 left) 2024-09-21T09:54:25.908 INFO:tasks.rados.rados.0.smithi033.stdout:1001: done (1 left) 2024-09-21T09:54:25.908 INFO:tasks.rados.rados.0.smithi033.stdout:1002: done (0 left) 2024-09-21T09:54:25.908 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 87 2024-09-21T09:54:25.908 INFO:tasks.rados.rados.0.smithi033.stdout:1003: seq_num 332 ranges {499712=770048} 2024-09-21T09:54:25.915 INFO:tasks.rados.rados.0.smithi033.stdout:1003: writing smithi03385152-37 from 499712 to 1269760 tid 1 2024-09-21T09:54:25.915 INFO:tasks.rados.rados.0.smithi033.stdout:1004: rmattr oid 2 current snap is 87 2024-09-21T09:54:25.915 INFO:tasks.rados.rados.0.smithi033.stdout:1004: done (1 left) 2024-09-21T09:54:25.915 INFO:tasks.rados.rados.0.smithi033.stdout:1005: read oid 2 snap 86 2024-09-21T09:54:25.915 INFO:tasks.rados.rados.0.smithi033.stdout:1005: expect (ObjNum 297 snap 76 seq_num 297) 2024-09-21T09:54:25.915 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 87 2024-09-21T09:54:25.915 INFO:tasks.rados.rados.0.smithi033.stdout:1006: seq_num 333 ranges {0=786432} 2024-09-21T09:54:25.920 INFO:tasks.rados.rados.0.smithi033.stdout:1006: writing smithi03385152-1 from 0 to 786432 tid 1 2024-09-21T09:54:25.920 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 87 2024-09-21T09:54:25.920 INFO:tasks.rados.rados.0.smithi033.stdout:1007: seq_num 334 ranges {2400256=802816} 2024-09-21T09:54:25.938 INFO:tasks.rados.rados.0.smithi033.stdout:1007: writing smithi03385152-32 from 2400256 to 3203072 tid 1 2024-09-21T09:54:25.938 INFO:tasks.rados.rados.0.smithi033.stdout:1008: rmattr oid 7 current snap is 87 2024-09-21T09:54:25.938 INFO:tasks.rados.rados.0.smithi033.stdout:1008: done (4 left) 2024-09-21T09:54:25.938 INFO:tasks.rados.rados.0.smithi033.stdout:1009: setattr oid 38 current snap is 87 2024-09-21T09:54:25.938 INFO:tasks.rados.rados.0.smithi033.stdout:1003: finishing write tid 1 to smithi03385152-37 2024-09-21T09:54:25.938 INFO:tasks.rados.rados.0.smithi033.stdout:1003: finishing write tid 2 to smithi03385152-37 2024-09-21T09:54:25.938 INFO:tasks.rados.rados.0.smithi033.stdout:1005: expect (ObjNum 297 snap 76 seq_num 297) 2024-09-21T09:54:26.005 INFO:tasks.rados.rados.0.smithi033.stdout:1003: finishing write tid 3 to smithi03385152-37 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 372 (ObjNum 332 snap 87 seq_num 332) dirty exists 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1003: left oid 37 (ObjNum 332 snap 87 seq_num 332) 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1006: finishing write tid 1 to smithi03385152-1 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1006: finishing write tid 2 to smithi03385152-1 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1006: finishing write tid 3 to smithi03385152-1 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 320 (ObjNum 333 snap 87 seq_num 333) dirty exists 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1006: left oid 1 (ObjNum 333 snap 87 seq_num 333) 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1007: finishing write tid 1 to smithi03385152-32 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1007: finishing write tid 2 to smithi03385152-32 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1007: finishing write tid 3 to smithi03385152-32 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 118 (ObjNum 334 snap 87 seq_num 334) dirty exists 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1007: left oid 32 (ObjNum 334 snap 87 seq_num 334) 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1003: done (4 left) 2024-09-21T09:54:26.006 INFO:tasks.rados.rados.0.smithi033.stdout:1005: done (3 left) 2024-09-21T09:54:26.007 INFO:tasks.rados.rados.0.smithi033.stdout:1006: done (2 left) 2024-09-21T09:54:26.007 INFO:tasks.rados.rados.0.smithi033.stdout:1007: done (1 left) 2024-09-21T09:54:26.007 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 87 2024-09-21T09:54:26.007 INFO:tasks.rados.rados.0.smithi033.stdout:1010: seq_num 335 ranges {4284416=417792,4702208=401408} 2024-09-21T09:54:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:25 smithi105 ceph-mon[31286]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:54:26.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:26.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:26.031 INFO:tasks.rados.rados.0.smithi033.stdout:1010: writing smithi03385152-15 from 4284416 to 4702208 tid 1 2024-09-21T09:54:26.033 INFO:tasks.rados.rados.0.smithi033.stdout:1010: writing smithi03385152-15 from 4702208 to 5103616 tid 2 2024-09-21T09:54:26.033 INFO:tasks.rados.rados.0.smithi033.stdout:1011: delete oid 4 current snap is 87 2024-09-21T09:54:26.033 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 615 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:54:26.033 INFO:tasks.rados.rados.0.smithi033.stdout:1010: finishing write tid 1 to smithi03385152-15 2024-09-21T09:54:26.036 INFO:tasks.rados.rados.0.smithi033.stdout:1010: finishing write tid 2 to smithi03385152-15 2024-09-21T09:54:26.036 INFO:tasks.rados.rados.0.smithi033.stdout:1010: finishing write tid 3 to smithi03385152-15 2024-09-21T09:54:26.037 INFO:tasks.rados.rados.0.smithi033.stdout:1010: finishing write tid 4 to smithi03385152-15 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 323 (ObjNum 335 snap 87 seq_num 335) dirty exists 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1010: left oid 15 (ObjNum 335 snap 87 seq_num 335) 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1009: done (2 left) 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1010: done (1 left) 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1011: done (0 left) 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1012: read oid 47 snap -1 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1012: expect deleted 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1013: rmattr oid 12 current snap is 87 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1013: done (1 left) 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1014: rmattr oid 1 current snap is 87 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1014: done (1 left) 2024-09-21T09:54:26.038 INFO:tasks.rados.rados.0.smithi033.stdout:1015: setattr oid 6 current snap is 87 2024-09-21T09:54:26.039 INFO:tasks.rados.rados.0.smithi033.stdout:1012: done (1 left) 2024-09-21T09:54:26.039 INFO:tasks.rados.rados.0.smithi033.stdout:1016: read oid 21 snap 86 2024-09-21T09:54:26.039 INFO:tasks.rados.rados.0.smithi033.stdout:1016: expect (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:54:26.039 INFO:tasks.rados.rados.0.smithi033.stdout:1017: delete oid 48 current snap is 87 2024-09-21T09:54:26.041 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 340 (ObjNum 172 snap 37 seq_num 172) dirty exists 2024-09-21T09:54:26.042 INFO:tasks.rados.rados.0.smithi033.stdout:1015: done (2 left) 2024-09-21T09:54:26.042 INFO:tasks.rados.rados.0.smithi033.stdout:1017: done (1 left) 2024-09-21T09:54:26.042 INFO:tasks.rados.rados.0.smithi033.stdout:1018: copy_from oid 11 from oid 22 current snap is 87 2024-09-21T09:54:26.042 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 87 2024-09-21T09:54:26.042 INFO:tasks.rados.rados.0.smithi033.stdout:1019: seq_num 338 ranges {4653056=466944,5120000=401408} 2024-09-21T09:54:26.070 INFO:tasks.rados.rados.0.smithi033.stdout:1019: writing smithi03385152-17 from 4653056 to 5120000 tid 1 2024-09-21T09:54:26.073 INFO:tasks.rados.rados.0.smithi033.stdout:1019: writing smithi03385152-17 from 5120000 to 5521408 tid 2 2024-09-21T09:54:26.073 INFO:tasks.rados.rados.0.smithi033.stdout:1020: setattr oid 29 current snap is 87 2024-09-21T09:54:26.073 INFO:tasks.rados.rados.0.smithi033.stdout:1018: finishing copy_from to smithi03385152-11 2024-09-21T09:54:26.073 INFO:tasks.rados.rados.0.smithi033.stdout:1018: got expected ENOENT (src dne) 2024-09-21T09:54:26.073 INFO:tasks.rados.rados.0.smithi033.stdout:1018: finishing copy_from racing read to smithi03385152-11 2024-09-21T09:54:26.073 INFO:tasks.rados.rados.0.smithi033.stdout:1016: expect (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:54:26.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:25 smithi033 ceph-mon[39045]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:54:26.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:25 smithi033 ceph-mon[96568]: osdmap e246: 8 total, 8 up, 8 in 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1019: finishing write tid 1 to smithi03385152-17 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1019: finishing write tid 2 to smithi03385152-17 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1019: finishing write tid 3 to smithi03385152-17 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1019: finishing write tid 4 to smithi03385152-17 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 416 (ObjNum 338 snap 87 seq_num 338) dirty exists 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1019: left oid 17 (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1016: done (3 left) 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1018: done (2 left) 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1019: done (1 left) 2024-09-21T09:54:26.424 INFO:tasks.rados.rados.0.smithi033.stdout:1021: snap_create 2024-09-21T09:54:26.426 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 648 (ObjNum 81 snap 0 seq_num 91222528) dirty exists 2024-09-21T09:54:26.913 INFO:tasks.rados.rados.0.smithi033.stdout:1020: done (1 left) 2024-09-21T09:54:26.913 INFO:tasks.rados.rados.0.smithi033.stdout:1021: done (0 left) 2024-09-21T09:54:26.913 INFO:tasks.rados.rados.0.smithi033.stdout:1022: snap_create 2024-09-21T09:54:27.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:26 smithi033 ceph-mon[39045]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:54:27.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:26 smithi033 ceph-mon[96568]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:54:27.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:26 smithi105 ceph-mon[31286]: osdmap e247: 8 total, 8 up, 8 in 2024-09-21T09:54:27.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:27.914 INFO:tasks.rados.rados.0.smithi033.stdout:1022: done (0 left) 2024-09-21T09:54:27.914 INFO:tasks.rados.rados.0.smithi033.stdout:1023: copy_from oid 10 from oid 16 current snap is 89 2024-09-21T09:54:27.914 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 89 2024-09-21T09:54:27.914 INFO:tasks.rados.rados.0.smithi033.stdout:1024: seq_num 339 ranges {0=483328,483328=401408} 2024-09-21T09:54:27.917 INFO:tasks.rados.rados.0.smithi033.stdout:1024: writing smithi03385152-5 from 0 to 483328 tid 1 2024-09-21T09:54:27.920 INFO:tasks.rados.rados.0.smithi033.stdout:1024: writing smithi03385152-5 from 483328 to 884736 tid 2 2024-09-21T09:54:27.920 INFO:tasks.rados.rados.0.smithi033.stdout:1025: read oid 23 snap 84 2024-09-21T09:54:27.920 INFO:tasks.rados.rados.0.smithi033.stdout:1025: expect (ObjNum 242 snap 63 seq_num 242) 2024-09-21T09:54:27.920 INFO:tasks.rados.rados.0.smithi033.stdout:1026: read oid 47 snap -1 2024-09-21T09:54:27.920 INFO:tasks.rados.rados.0.smithi033.stdout:1026: expect deleted 2024-09-21T09:54:27.920 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 89 2024-09-21T09:54:27.920 INFO:tasks.rados.rados.0.smithi033.stdout:1027: seq_num 340 ranges {5103616=499712,5603328=401408} 2024-09-21T09:54:27.951 INFO:tasks.rados.rados.0.smithi033.stdout:1027: writing smithi03385152-15 from 5103616 to 5603328 tid 1 2024-09-21T09:54:27.953 INFO:tasks.rados.rados.0.smithi033.stdout:1027: writing smithi03385152-15 from 5603328 to 6004736 tid 2 2024-09-21T09:54:27.953 INFO:tasks.rados.rados.0.smithi033.stdout:1028: snap_remove snap 86 2024-09-21T09:54:28.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:27 smithi033 ceph-mon[39045]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:54:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:27 smithi033 ceph-mon[96568]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:54:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:27 smithi105 ceph-mon[31286]: osdmap e248: 8 total, 8 up, 8 in 2024-09-21T09:54:28.925 INFO:tasks.rados.rados.0.smithi033.stdout:1028: done (5 left) 2024-09-21T09:54:28.925 INFO:tasks.rados.rados.0.smithi033.stdout:1029: setattr oid 7 current snap is 89 2024-09-21T09:54:28.925 INFO:tasks.rados.rados.0.smithi033.stdout:1024: finishing write tid 1 to smithi03385152-5 2024-09-21T09:54:28.925 INFO:tasks.rados.rados.0.smithi033.stdout:1024: finishing write tid 2 to smithi03385152-5 2024-09-21T09:54:28.925 INFO:tasks.rados.rados.0.smithi033.stdout:1024: finishing write tid 3 to smithi03385152-5 2024-09-21T09:54:28.925 INFO:tasks.rados.rados.0.smithi033.stdout:1024: finishing write tid 4 to smithi03385152-5 2024-09-21T09:54:28.925 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 111 (ObjNum 339 snap 89 seq_num 339) dirty exists 2024-09-21T09:54:28.925 INFO:tasks.rados.rados.0.smithi033.stdout:1024: left oid 5 (ObjNum 339 snap 89 seq_num 339) 2024-09-21T09:54:28.926 INFO:tasks.rados.rados.0.smithi033.stdout:1023: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:54:28.926 INFO:tasks.rados.rados.0.smithi033.stdout:1023: finishing copy_from to smithi03385152-10 2024-09-21T09:54:28.926 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 631 (ObjNum 250 snap 65 seq_num 250) dirty exists 2024-09-21T09:54:28.926 INFO:tasks.rados.rados.0.smithi033.stdout:1025: expect (ObjNum 242 snap 63 seq_num 242) 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:1027: finishing write tid 1 to smithi03385152-15 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:1027: finishing write tid 2 to smithi03385152-15 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:1027: finishing write tid 3 to smithi03385152-15 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:1027: finishing write tid 4 to smithi03385152-15 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 326 (ObjNum 340 snap 89 seq_num 340) dirty exists 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:1027: left oid 15 (ObjNum 340 snap 89 seq_num 340) 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:1023: done (5 left) 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:1024: done (4 left) 2024-09-21T09:54:29.062 INFO:tasks.rados.rados.0.smithi033.stdout:1025: done (3 left) 2024-09-21T09:54:29.063 INFO:tasks.rados.rados.0.smithi033.stdout:1026: done (2 left) 2024-09-21T09:54:29.063 INFO:tasks.rados.rados.0.smithi033.stdout:1027: done (1 left) 2024-09-21T09:54:29.063 INFO:tasks.rados.rados.0.smithi033.stdout:1030: rmattr oid 42 current snap is 89 2024-09-21T09:54:29.063 INFO:tasks.rados.rados.0.smithi033.stdout:1031: snap_remove snap 87 2024-09-21T09:54:29.181 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:28 smithi105 ceph-mon[31286]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:54:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:28 smithi033 ceph-mon[39045]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:54:29.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:28 smithi033 ceph-mon[96568]: osdmap e249: 8 total, 8 up, 8 in 2024-09-21T09:54:29.944 INFO:tasks.rados.rados.0.smithi033.stdout:1031: done (2 left) 2024-09-21T09:54:29.944 INFO:tasks.rados.rados.0.smithi033.stdout:1032: rollback oid 5 current snap is 89 2024-09-21T09:54:29.944 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 5 to 88 2024-09-21T09:54:29.944 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 616 (ObjNum 54 snap 4 seq_num 54) dirty exists 2024-09-21T09:54:29.944 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 652 (ObjNum 324 snap 84 seq_num 324) dirty exists 2024-09-21T09:54:29.945 INFO:tasks.rados.rados.0.smithi033.stdout:1029: done (2 left) 2024-09-21T09:54:29.945 INFO:tasks.rados.rados.0.smithi033.stdout:1030: done (1 left) 2024-09-21T09:54:29.945 INFO:tasks.rados.rados.0.smithi033.stdout:1033: delete oid 49 current snap is 89 2024-09-21T09:54:29.948 INFO:tasks.rados.rados.0.smithi033.stdout:1033: done (1 left) 2024-09-21T09:54:29.948 INFO:tasks.rados.rados.0.smithi033.stdout:1034: delete oid 20 current snap is 89 2024-09-21T09:54:29.951 INFO:tasks.rados.rados.0.smithi033.stdout:1032: finishing rollback tid 0 to smithi03385152-5 2024-09-21T09:54:29.953 INFO:tasks.rados.rados.0.smithi033.stdout:1032: finishing rollback tid 1 to smithi03385152-5 2024-09-21T09:54:29.953 INFO:tasks.rados.rados.0.smithi033.stdout:1032: finishing rollback tid 2 to smithi03385152-5 2024-09-21T09:54:29.953 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 116 (ObjNum 97 snap 0 seq_num 91256688) dirty exists 2024-09-21T09:54:29.954 INFO:tasks.rados.rados.0.smithi033.stdout:1032: done (1 left) 2024-09-21T09:54:29.954 INFO:tasks.rados.rados.0.smithi033.stdout:1034: done (0 left) 2024-09-21T09:54:29.954 INFO:tasks.rados.rados.0.smithi033.stdout:1035: snap_remove snap 84 2024-09-21T09:54:30.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-mon[31286]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:54:30.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-mon[31286]: Standby manager daemon x restarted 2024-09-21T09:54:30.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-mon[31286]: Standby manager daemon x started 2024-09-21T09:54:30.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-mon[31286]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: [21/Sep/2024:09:54:29] ENGINE Bus STARTING 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: CherryPy Checker: 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: The Application mounted at '' has an empty config. 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: [21/Sep/2024:09:54:30] ENGINE Serving on http://:::9283 2024-09-21T09:54:30.215 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x[79256]: [21/Sep/2024:09:54:30] ENGINE Bus STARTED 2024-09-21T09:54:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[39045]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:54:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[39045]: Standby manager daemon x restarted 2024-09-21T09:54:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[39045]: Standby manager daemon x started 2024-09-21T09:54:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[39045]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:30.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[96568]: osdmap e250: 8 total, 8 up, 8 in 2024-09-21T09:54:30.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[96568]: Standby manager daemon x restarted 2024-09-21T09:54:30.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[96568]: Standby manager daemon x started 2024-09-21T09:54:30.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[96568]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-21T09:54:30.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[96568]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-21T09:54:30.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[96568]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-21T09:54:30.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:29 smithi033 ceph-mon[96568]: from='mgr.? 172.21.15.105:0/256785581' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-21T09:54:30.297 INFO:tasks.rados.rados.0.smithi033.stdout:1035: done (0 left) 2024-09-21T09:54:30.297 INFO:tasks.rados.rados.0.smithi033.stdout:1036: snap_remove snap 88 2024-09-21T09:54:30.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.815+0000 7fc35ede8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.815+0000 7fc35ede8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.815+0000 7fc35ede8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.815+0000 7fc35ede8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.815+0000 7fc35ede8640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.817+0000 7fc35c5e3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.817+0000 7fc35c5e3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.817+0000 7fc35c5e3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.817+0000 7fc35c5e3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.152 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:30.817+0000 7fc35c5e3640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: mgrmap e38: y(active, since 39s), standbys: x 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: Active manager daemon y restarted 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: Activating manager daemon y 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: mgrmap e39: y(active, starting, since 0.00736852s), standbys: x 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:31.153 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:54:31.154 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: Manager daemon y is now available 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:54:31.155 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: mgrmap e38: y(active, since 39s), standbys: x 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: Active manager daemon y restarted 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: Activating manager daemon y 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: mgrmap e39: y(active, starting, since 0.00736852s), standbys: x 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:31.156 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:54:31.157 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:54:31.158 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: Manager daemon y is now available 2024-09-21T09:54:31.158 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:31.158 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:31.158 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:54:31.158 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:54:31.158 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:54:31.158 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:54:31.158 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:30 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:54:31.185 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: osdmap e251: 8 total, 8 up, 8 in 2024-09-21T09:54:31.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: mgrmap e38: y(active, since 39s), standbys: x 2024-09-21T09:54:31.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: Active manager daemon y restarted 2024-09-21T09:54:31.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: Activating manager daemon y 2024-09-21T09:54:31.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: osdmap e252: 8 total, 8 up, 8 in 2024-09-21T09:54:31.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: mgrmap e39: y(active, starting, since 0.00736852s), standbys: x 2024-09-21T09:54:31.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:31.187 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:31.187 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:31.187 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "a"}]: dispatch 2024-09-21T09:54:31.187 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "b"}]: dispatch 2024-09-21T09:54:31.187 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-21T09:54:31.187 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-21T09:54:31.187 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-21T09:54:31.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-21T09:54:31.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-21T09:54:31.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-21T09:54:31.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-21T09:54:31.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-21T09:54:31.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-21T09:54:31.188 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-21T09:54:31.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-21T09:54:31.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-21T09:54:31.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-21T09:54:31.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: Manager daemon y is now available 2024-09-21T09:54:31.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:31.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:54:31.189 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:54:31.190 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-21T09:54:31.190 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-21T09:54:31.190 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-21T09:54:31.190 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:30 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-21T09:54:31.313 INFO:tasks.rados.rados.0.smithi033.stdout:1036: done (0 left) 2024-09-21T09:54:31.313 INFO:tasks.rados.rados.0.smithi033.stdout:1037: snap_create 2024-09-21T09:54:31.419 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:31.292+0000 7fc38e955640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-21T09:54:31.420 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:31] ENGINE Bus STARTING 2024-09-21T09:54:31.420 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: CherryPy Checker: 2024-09-21T09:54:31.420 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: The Application mounted at '' has an empty config. 2024-09-21T09:54:31.420 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:31.420 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:31] ENGINE Serving on http://:::9283 2024-09-21T09:54:31.420 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:31] ENGINE Bus STARTED 2024-09-21T09:54:31.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:54:31] "GET /metrics HTTP/1.1" 200 33458 "" "Prometheus/2.51.0" 2024-09-21T09:54:32.317 INFO:tasks.rados.rados.0.smithi033.stdout:1037: done (0 left) 2024-09-21T09:54:32.318 INFO:tasks.rados.rados.0.smithi033.stdout:1038: rollback oid 48 current snap is 90 2024-09-21T09:54:32.318 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 48 to 89 2024-09-21T09:54:32.318 INFO:tasks.rados.rados.0.smithi033.stdout:1039: copy_from oid 4 from oid 24 current snap is 90 2024-09-21T09:54:32.318 INFO:tasks.rados.rados.0.smithi033.stdout:1040: snap_create 2024-09-21T09:54:32.320 INFO:tasks.rados.rados.0.smithi033.stdout:1038: finishing rollback tid 1 to smithi03385152-48 2024-09-21T09:54:32.320 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 0 (ObjNum 1089 snap 0 seq_num 1163895008) dirty dne 2024-09-21T09:54:32.359 INFO:tasks.rados.rados.0.smithi033.stdout:1039: finishing copy_from racing read to smithi03385152-4 2024-09-21T09:54:32.363 INFO:tasks.rados.rados.0.smithi033.stdout:1039: finishing copy_from to smithi03385152-4 2024-09-21T09:54:32.363 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 157 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:54:32.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:32 smithi105 ceph-mon[31286]: mgrmap e40: y(active, since 1.01059s), standbys: x 2024-09-21T09:54:32.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:32 smithi105 ceph-mon[31286]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:54:32.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:32 smithi105 ceph-mon[31286]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:54:32.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:32 smithi105 ceph-mon[31286]: pgmap v4: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 GiB/s rd, 19 GiB/s wr, 172.31k op/s 2024-09-21T09:54:32.605 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-mon[39045]: mgrmap e40: y(active, since 1.01059s), standbys: x 2024-09-21T09:54:32.605 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-mon[39045]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:54:32.605 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-mon[39045]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:54:32.605 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-mon[39045]: pgmap v4: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 GiB/s rd, 19 GiB/s wr, 172.31k op/s 2024-09-21T09:54:32.605 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-mon[96568]: mgrmap e40: y(active, since 1.01059s), standbys: x 2024-09-21T09:54:32.605 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-mon[96568]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-21T09:54:32.605 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-mon[96568]: osdmap e253: 8 total, 8 up, 8 in 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-mon[96568]: pgmap v4: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 52 GiB/s rd, 19 GiB/s wr, 172.31k op/s 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:32.345+0000 7fc342435640 -1 log_channel(cephadm) log [ERR] : [21/Sep/2024:09:54:32] ENGINE Error in HTTPServer.serve 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: Traceback (most recent call last): 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: self._connections.run(self.expiration_interval) 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: self._run(expiration_interval) 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:54:32.606 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: s = self.context.wrap_socket( 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: return self.sslsocket_class._create( 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: self.do_handshake() 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: self._sslobj.do_handshake() 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:54:32.607 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:33.316 INFO:tasks.rados.rados.0.smithi033.stdout:1038: done (2 left) 2024-09-21T09:54:33.316 INFO:tasks.rados.rados.0.smithi033.stdout:1039: done (1 left) 2024-09-21T09:54:33.317 INFO:tasks.rados.rados.0.smithi033.stdout:1040: done (0 left) 2024-09-21T09:54:33.317 INFO:tasks.rados.rados.0.smithi033.stdout:1041: read oid 34 snap 89 2024-09-21T09:54:33.317 INFO:tasks.rados.rados.0.smithi033.stdout:1041: expect (ObjNum 313 snap 83 seq_num 313) 2024-09-21T09:54:33.317 INFO:tasks.rados.rados.0.smithi033.stdout:1042: rollback oid 27 current snap is 91 2024-09-21T09:54:33.317 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 27 to 89 2024-09-21T09:54:33.317 INFO:tasks.rados.rados.0.smithi033.stdout:1043: delete oid 13 current snap is 91 2024-09-21T09:54:33.327 INFO:tasks.rados.rados.0.smithi033.stdout:1042: finishing rollback tid 0 to smithi03385152-27 2024-09-21T09:54:33.328 INFO:tasks.rados.rados.0.smithi033.stdout:1043: done (2 left) 2024-09-21T09:54:33.328 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 91 2024-09-21T09:54:33.328 INFO:tasks.rados.rados.0.smithi033.stdout:1044: seq_num 344 ranges {0=565248,565248=401408} 2024-09-21T09:54:33.331 INFO:tasks.rados.rados.0.smithi033.stdout:1044: writing smithi03385152-31 from 0 to 565248 tid 1 2024-09-21T09:54:33.333 INFO:tasks.rados.rados.0.smithi033.stdout:1044: writing smithi03385152-31 from 565248 to 966656 tid 2 2024-09-21T09:54:33.333 INFO:tasks.rados.rados.0.smithi033.stdout:1045: read oid 8 snap -1 2024-09-21T09:54:33.333 INFO:tasks.rados.rados.0.smithi033.stdout:1045: expect (ObjNum 290 snap 70 seq_num 290) 2024-09-21T09:54:33.334 INFO:tasks.rados.rados.0.smithi033.stdout:1046: snap_remove snap 90 2024-09-21T09:54:33.429 INFO:tasks.rados.rados.0.smithi033.stdout:1046: done (4 left) 2024-09-21T09:54:33.429 INFO:tasks.rados.rados.0.smithi033.stdout:1047: read oid 10 snap -1 2024-09-21T09:54:33.429 INFO:tasks.rados.rados.0.smithi033.stdout:1047: expect (ObjNum 250 snap 65 seq_num 250) 2024-09-21T09:54:33.429 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 91 2024-09-21T09:54:33.429 INFO:tasks.rados.rados.0.smithi033.stdout:1042: finishing rollback tid 1 to smithi03385152-27 2024-09-21T09:54:33.430 INFO:tasks.rados.rados.0.smithi033.stdout:1042: finishing rollback tid 2 to smithi03385152-27 2024-09-21T09:54:33.430 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 350 (ObjNum 325 snap 84 seq_num 325) dirty exists 2024-09-21T09:54:33.430 INFO:tasks.rados.rados.0.smithi033.stdout:1041: expect (ObjNum 313 snap 83 seq_num 313) 2024-09-21T09:54:33.450 INFO:tasks.rados.rados.0.smithi033.stdout:1044: finishing write tid 1 to smithi03385152-31 2024-09-21T09:54:33.451 INFO:tasks.rados.rados.0.smithi033.stdout:1044: finishing write tid 2 to smithi03385152-31 2024-09-21T09:54:33.451 INFO:tasks.rados.rados.0.smithi033.stdout:1044: finishing write tid 3 to smithi03385152-31 2024-09-21T09:54:33.451 INFO:tasks.rados.rados.0.smithi033.stdout:1044: finishing write tid 4 to smithi03385152-31 2024-09-21T09:54:33.451 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 671 (ObjNum 344 snap 91 seq_num 344) dirty exists 2024-09-21T09:54:33.451 INFO:tasks.rados.rados.0.smithi033.stdout:1044: left oid 31 (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:54:33.451 INFO:tasks.rados.rados.0.smithi033.stdout:1048: seq_num 345 ranges {2170880=581632,2752512=401408} 2024-09-21T09:54:33.465 INFO:tasks.rados.rados.0.smithi033.stdout:1048: writing smithi03385152-26 from 2170880 to 2752512 tid 1 2024-09-21T09:54:33.467 INFO:tasks.rados.rados.0.smithi033.stdout:1048: writing smithi03385152-26 from 2752512 to 3153920 tid 2 2024-09-21T09:54:33.468 INFO:tasks.rados.rados.0.smithi033.stdout:1041: done (5 left) 2024-09-21T09:54:33.468 INFO:tasks.rados.rados.0.smithi033.stdout:1042: done (4 left) 2024-09-21T09:54:33.468 INFO:tasks.rados.rados.0.smithi033.stdout:1044: done (3 left) 2024-09-21T09:54:33.468 INFO:tasks.rados.rados.0.smithi033.stdout:1049: copy_from oid 46 from oid 31 current snap is 91 2024-09-21T09:54:33.468 INFO:tasks.rados.rados.0.smithi033.stdout:1045: expect (ObjNum 290 snap 70 seq_num 290) 2024-09-21T09:54:33.584 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[39045]: [21/Sep/2024:09:54:32] ENGINE Bus STARTING 2024-09-21T09:54:33.584 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[39045]: pgmap v5: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 407 MiB/s rd, 150 MiB/s wr, 1.32k op/s 2024-09-21T09:54:33.584 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[39045]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:54:33.584 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[39045]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-21T09:54:33.584 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[39045]: [21/Sep/2024:09:54:32] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[39045]: [21/Sep/2024:09:54:32] ENGINE Error in HTTPServer.serve 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:54:33.585 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.c.smithi033.stdout: self.do_handshake() 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.c.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.c.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.c.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[39045]: [21/Sep/2024:09:54:32] ENGINE Serving on http://172.21.15.33:8765 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[39045]: [21/Sep/2024:09:54:32] ENGINE Bus STARTED 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[96568]: [21/Sep/2024:09:54:32] ENGINE Bus STARTING 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[96568]: pgmap v5: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 407 MiB/s rd, 150 MiB/s wr, 1.32k op/s 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[96568]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[96568]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[96568]: [21/Sep/2024:09:54:32] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:54:33.586 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[96568]: [21/Sep/2024:09:54:32] ENGINE Error in HTTPServer.serve 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: Traceback (most recent call last): 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._run(expiration_interval) 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: s = self.context.wrap_socket( 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: return self.sslsocket_class._create( 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:54:33.587 INFO:journalctl@ceph.mon.a.smithi033.stdout: self.do_handshake() 2024-09-21T09:54:33.588 INFO:journalctl@ceph.mon.a.smithi033.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:54:33.588 INFO:journalctl@ceph.mon.a.smithi033.stdout: self._sslobj.do_handshake() 2024-09-21T09:54:33.588 INFO:journalctl@ceph.mon.a.smithi033.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:54:33.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[96568]: [21/Sep/2024:09:54:32] ENGINE Serving on http://172.21.15.33:8765 2024-09-21T09:54:33.588 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:33 smithi033 ceph-mon[96568]: [21/Sep/2024:09:54:32] ENGINE Bus STARTED 2024-09-21T09:54:33.626 INFO:tasks.rados.rados.0.smithi033.stdout:1047: expect (ObjNum 250 snap 65 seq_num 250) 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:33 smithi105 ceph-mon[31286]: [21/Sep/2024:09:54:32] ENGINE Bus STARTING 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:33 smithi105 ceph-mon[31286]: pgmap v5: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 407 MiB/s rd, 150 MiB/s wr, 1.32k op/s 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:33 smithi105 ceph-mon[31286]: osdmap e254: 8 total, 8 up, 8 in 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:33 smithi105 ceph-mon[31286]: mgrmap e41: y(active, since 2s), standbys: x 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:33 smithi105 ceph-mon[31286]: [21/Sep/2024:09:54:32] ENGINE Serving on https://172.21.15.33:7150 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:33 smithi105 ceph-mon[31286]: [21/Sep/2024:09:54:32] ENGINE Error in HTTPServer.serve 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout: Traceback (most recent call last): 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._connections.run(self.expiration_interval) 2024-09-21T09:54:33.665 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._run(expiration_interval) 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: s = self.context.wrap_socket( 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: return self.sslsocket_class._create( 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: self.do_handshake() 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: self._sslobj.do_handshake() 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-21T09:54:33.666 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:33 smithi105 ceph-mon[31286]: [21/Sep/2024:09:54:32] ENGINE Serving on http://172.21.15.33:8765 2024-09-21T09:54:33.667 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:33 smithi105 ceph-mon[31286]: [21/Sep/2024:09:54:32] ENGINE Bus STARTED 2024-09-21T09:54:33.678 INFO:tasks.rados.rados.0.smithi033.stdout:1048: finishing write tid 1 to smithi03385152-26 2024-09-21T09:54:33.678 INFO:tasks.rados.rados.0.smithi033.stdout:1048: finishing write tid 2 to smithi03385152-26 2024-09-21T09:54:33.678 INFO:tasks.rados.rados.0.smithi033.stdout:1048: finishing write tid 3 to smithi03385152-26 2024-09-21T09:54:33.678 INFO:tasks.rados.rados.0.smithi033.stdout:1048: finishing write tid 4 to smithi03385152-26 2024-09-21T09:54:33.678 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 430 (ObjNum 345 snap 91 seq_num 345) dirty exists 2024-09-21T09:54:33.678 INFO:tasks.rados.rados.0.smithi033.stdout:1048: left oid 26 (ObjNum 345 snap 91 seq_num 345) 2024-09-21T09:54:33.678 INFO:tasks.rados.rados.0.smithi033.stdout:1049: finishing copy_from racing read to smithi03385152-46 2024-09-21T09:54:33.678 INFO:tasks.rados.rados.0.smithi033.stdout:1049: finishing copy_from to smithi03385152-46 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 671 (ObjNum 344 snap 91 seq_num 344) dirty exists 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1045: done (3 left) 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1047: done (2 left) 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1048: done (1 left) 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1049: done (0 left) 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1050: read oid 26 snap -1 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1050: expect (ObjNum 345 snap 91 seq_num 345) 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1051: read oid 31 snap -1 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1051: expect (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 91 2024-09-21T09:54:33.679 INFO:tasks.rados.rados.0.smithi033.stdout:1052: seq_num 346 ranges {0=598016,598016=401408} 2024-09-21T09:54:33.681 INFO:tasks.rados.rados.0.smithi033.stdout:1052: writing smithi03385152-3 from 0 to 598016 tid 1 2024-09-21T09:54:33.683 INFO:tasks.rados.rados.0.smithi033.stdout:1052: writing smithi03385152-3 from 598016 to 999424 tid 2 2024-09-21T09:54:33.684 INFO:tasks.rados.rados.0.smithi033.stdout:1053: read oid 40 snap -1 2024-09-21T09:54:33.684 INFO:tasks.rados.rados.0.smithi033.stdout:1053: expect (ObjNum 114 snap 25 seq_num 114) 2024-09-21T09:54:33.684 INFO:tasks.rados.rados.0.smithi033.stdout:1054: setattr oid 5 current snap is 91 2024-09-21T09:54:33.684 INFO:tasks.rados.rados.0.smithi033.stdout:1051: expect (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:54:33.713 INFO:tasks.rados.rados.0.smithi033.stdout:1052: finishing write tid 1 to smithi03385152-3 2024-09-21T09:54:33.713 INFO:tasks.rados.rados.0.smithi033.stdout:1052: finishing write tid 2 to smithi03385152-3 2024-09-21T09:54:33.713 INFO:tasks.rados.rados.0.smithi033.stdout:1052: finishing write tid 3 to smithi03385152-3 2024-09-21T09:54:33.713 INFO:tasks.rados.rados.0.smithi033.stdout:1052: finishing write tid 4 to smithi03385152-3 2024-09-21T09:54:33.714 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 674 (ObjNum 346 snap 91 seq_num 346) dirty exists 2024-09-21T09:54:33.714 INFO:tasks.rados.rados.0.smithi033.stdout:1052: left oid 3 (ObjNum 346 snap 91 seq_num 346) 2024-09-21T09:54:33.714 INFO:tasks.rados.rados.0.smithi033.stdout:1053: expect (ObjNum 114 snap 25 seq_num 114) 2024-09-21T09:54:33.797 INFO:tasks.rados.rados.0.smithi033.stdout:1050: expect (ObjNum 345 snap 91 seq_num 345) 2024-09-21T09:54:33.920 INFO:tasks.rados.rados.0.smithi033.stdout:1050: done (4 left) 2024-09-21T09:54:33.920 INFO:tasks.rados.rados.0.smithi033.stdout:1051: done (3 left) 2024-09-21T09:54:33.920 INFO:tasks.rados.rados.0.smithi033.stdout:1052: done (2 left) 2024-09-21T09:54:33.920 INFO:tasks.rados.rados.0.smithi033.stdout:1053: done (1 left) 2024-09-21T09:54:33.920 INFO:tasks.rados.rados.0.smithi033.stdout:1055: read oid 46 snap 89 2024-09-21T09:54:33.920 INFO:tasks.rados.rados.0.smithi033.stdout:1055: expect (ObjNum 253 snap 65 seq_num 253) 2024-09-21T09:54:33.921 INFO:tasks.rados.rados.0.smithi033.stdout:1056: snap_create 2024-09-21T09:54:33.922 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 124 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:54:33.928 INFO:tasks.rados.rados.0.smithi033.stdout:1055: expect (ObjNum 253 snap 65 seq_num 253) 2024-09-21T09:54:34.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:34.444 INFO:tasks.rados.rados.0.smithi033.stdout:1054: done (2 left) 2024-09-21T09:54:34.444 INFO:tasks.rados.rados.0.smithi033.stdout:1055: done (1 left) 2024-09-21T09:54:34.445 INFO:tasks.rados.rados.0.smithi033.stdout:1056: done (0 left) 2024-09-21T09:54:34.445 INFO:tasks.rados.rados.0.smithi033.stdout:1057: read oid 50 snap -1 2024-09-21T09:54:34.445 INFO:tasks.rados.rados.0.smithi033.stdout:1057: expect (ObjNum 289 snap 70 seq_num 289) 2024-09-21T09:54:34.445 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 92 2024-09-21T09:54:34.445 INFO:tasks.rados.rados.0.smithi033.stdout:1058: seq_num 347 ranges {0=614400,614400=401408} 2024-09-21T09:54:34.448 INFO:tasks.rados.rados.0.smithi033.stdout:1058: writing smithi03385152-49 from 0 to 614400 tid 1 2024-09-21T09:54:34.451 INFO:tasks.rados.rados.0.smithi033.stdout:1058: writing smithi03385152-49 from 614400 to 1015808 tid 2 2024-09-21T09:54:34.451 INFO:tasks.rados.rados.0.smithi033.stdout:1059: rollback oid 32 current snap is 92 2024-09-21T09:54:34.451 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 32 to 89 2024-09-21T09:54:34.451 INFO:tasks.rados.rados.0.smithi033.stdout:1060: delete oid 37 current snap is 92 2024-09-21T09:54:34.454 INFO:tasks.rados.rados.0.smithi033.stdout:1058: finishing write tid 1 to smithi03385152-49 2024-09-21T09:54:34.454 INFO:tasks.rados.rados.0.smithi033.stdout:1058: finishing write tid 2 to smithi03385152-49 2024-09-21T09:54:34.454 INFO:tasks.rados.rados.0.smithi033.stdout:1060: done (3 left) 2024-09-21T09:54:34.454 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 92 2024-09-21T09:54:34.455 INFO:tasks.rados.rados.0.smithi033.stdout:1061: seq_num 349 ranges {3227648=655360,3883008=401408} 2024-09-21T09:54:34.476 INFO:tasks.rados.rados.0.smithi033.stdout:1061: writing smithi03385152-27 from 3227648 to 3883008 tid 1 2024-09-21T09:54:34.479 INFO:tasks.rados.rados.0.smithi033.stdout:1061: writing smithi03385152-27 from 3883008 to 4284416 tid 2 2024-09-21T09:54:34.479 INFO:tasks.rados.rados.0.smithi033.stdout:1062: delete oid 38 current snap is 92 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:1058: finishing write tid 3 to smithi03385152-49 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:1058: finishing write tid 4 to smithi03385152-49 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 433 (ObjNum 347 snap 92 seq_num 347) dirty exists 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:1058: left oid 49 (ObjNum 347 snap 92 seq_num 347) 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:1059: finishing rollback tid 0 to smithi03385152-32 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:1059: finishing rollback tid 1 to smithi03385152-32 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:1059: finishing rollback tid 2 to smithi03385152-32 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 126 (ObjNum 334 snap 87 seq_num 334) dirty exists 2024-09-21T09:54:34.480 INFO:tasks.rados.rados.0.smithi033.stdout:1057: expect (ObjNum 289 snap 70 seq_num 289) 2024-09-21T09:54:34.612 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:34 smithi105 ceph-mon[31286]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:54:34.612 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:34 smithi105 ceph-mon[31286]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:54:34.656 INFO:tasks.rados.rados.0.smithi033.stdout:1061: finishing write tid 1 to smithi03385152-27 2024-09-21T09:54:34.656 INFO:tasks.rados.rados.0.smithi033.stdout:1061: finishing write tid 2 to smithi03385152-27 2024-09-21T09:54:34.656 INFO:tasks.rados.rados.0.smithi033.stdout:1061: finishing write tid 3 to smithi03385152-27 2024-09-21T09:54:34.656 INFO:tasks.rados.rados.0.smithi033.stdout:1061: finishing write tid 4 to smithi03385152-27 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 356 (ObjNum 349 snap 92 seq_num 349) dirty exists 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:1061: left oid 27 (ObjNum 349 snap 92 seq_num 349) 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:1057: done (4 left) 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:1058: done (3 left) 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:1059: done (2 left) 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:1061: done (1 left) 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:1062: done (0 left) 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 92 2024-09-21T09:54:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:1063: seq_num 351 ranges {2695168=688128,3383296=401408} 2024-09-21T09:54:34.674 INFO:tasks.rados.rados.0.smithi033.stdout:1063: writing smithi03385152-9 from 2695168 to 3383296 tid 1 2024-09-21T09:54:34.677 INFO:tasks.rados.rados.0.smithi033.stdout:1063: writing smithi03385152-9 from 3383296 to 3784704 tid 2 2024-09-21T09:54:34.677 INFO:tasks.rados.rados.0.smithi033.stdout:1064: read oid 11 snap -1 2024-09-21T09:54:34.677 INFO:tasks.rados.rados.0.smithi033.stdout:1064: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:54:34.677 INFO:tasks.rados.rados.0.smithi033.stdout:1065: snap_create 2024-09-21T09:54:34.679 INFO:tasks.rados.rados.0.smithi033.stdout:1063: finishing write tid 1 to smithi03385152-9 2024-09-21T09:54:34.679 INFO:tasks.rados.rados.0.smithi033.stdout:1063: finishing write tid 2 to smithi03385152-9 2024-09-21T09:54:34.680 INFO:tasks.rados.rados.0.smithi033.stdout:1063: finishing write tid 3 to smithi03385152-9 2024-09-21T09:54:34.681 INFO:tasks.rados.rados.0.smithi033.stdout:1063: finishing write tid 4 to smithi03385152-9 2024-09-21T09:54:34.681 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 620 (ObjNum 351 snap 92 seq_num 351) dirty exists 2024-09-21T09:54:34.681 INFO:tasks.rados.rados.0.smithi033.stdout:1063: left oid 9 (ObjNum 351 snap 92 seq_num 351) 2024-09-21T09:54:34.686 INFO:tasks.rados.rados.0.smithi033.stdout:1064: expect (ObjNum 192 snap 45 seq_num 192) 2024-09-21T09:54:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:34 smithi033 ceph-mon[39045]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:54:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:34 smithi033 ceph-mon[39045]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:54:34.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:34 smithi033 ceph-mon[96568]: osdmap e255: 8 total, 8 up, 8 in 2024-09-21T09:54:34.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:34 smithi033 ceph-mon[96568]: osdmap e256: 8 total, 8 up, 8 in 2024-09-21T09:54:35.449 INFO:tasks.rados.rados.0.smithi033.stdout:1063: done (2 left) 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:1064: done (1 left) 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:1065: done (0 left) 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:1066: read oid 12 snap -1 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:1066: expect deleted 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:1067: rollback oid 47 current snap is 93 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 89 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:1068: copy_from oid 22 from oid 35 current snap is 93 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:1069: delete oid 3 current snap is 93 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:1067: finishing rollback tid 1 to smithi03385152-47 2024-09-21T09:54:35.450 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 0 (ObjNum 33 snap 0 seq_num 72673360) dirty dne 2024-09-21T09:54:35.455 INFO:tasks.rados.rados.0.smithi033.stdout:1066: done (3 left) 2024-09-21T09:54:35.455 INFO:tasks.rados.rados.0.smithi033.stdout:1067: done (2 left) 2024-09-21T09:54:35.455 INFO:tasks.rados.rados.0.smithi033.stdout:1069: done (1 left) 2024-09-21T09:54:35.455 INFO:tasks.rados.rados.0.smithi033.stdout:1070: rmattr oid 34 current snap is 93 2024-09-21T09:54:35.455 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 93 2024-09-21T09:54:35.456 INFO:tasks.rados.rados.0.smithi033.stdout:1071: seq_num 353 ranges {4161536=720896,4882432=401408} 2024-09-21T09:54:35.481 INFO:tasks.rados.rados.0.smithi033.stdout:1071: writing smithi03385152-6 from 4161536 to 4882432 tid 1 2024-09-21T09:54:35.484 INFO:tasks.rados.rados.0.smithi033.stdout:1071: writing smithi03385152-6 from 4882432 to 5283840 tid 2 2024-09-21T09:54:35.484 INFO:tasks.rados.rados.0.smithi033.stdout:1072: snap_create 2024-09-21T09:54:35.484 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 676 (ObjNum 313 snap 83 seq_num 313) dirty exists 2024-09-21T09:54:35.484 INFO:tasks.rados.rados.0.smithi033.stdout:1068: finishing copy_from racing read to smithi03385152-22 2024-09-21T09:54:35.484 INFO:tasks.rados.rados.0.smithi033.stdout:1068: finishing copy_from to smithi03385152-22 2024-09-21T09:54:35.484 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 642 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:54:35.485 INFO:tasks.rados.rados.0.smithi033.stdout:1071: finishing write tid 1 to smithi03385152-6 2024-09-21T09:54:35.486 INFO:tasks.rados.rados.0.smithi033.stdout:1071: finishing write tid 2 to smithi03385152-6 2024-09-21T09:54:35.486 INFO:tasks.rados.rados.0.smithi033.stdout:1071: finishing write tid 3 to smithi03385152-6 2024-09-21T09:54:35.489 INFO:tasks.rados.rados.0.smithi033.stdout:1071: finishing write tid 4 to smithi03385152-6 2024-09-21T09:54:35.489 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 360 (ObjNum 353 snap 93 seq_num 353) dirty exists 2024-09-21T09:54:35.489 INFO:tasks.rados.rados.0.smithi033.stdout:1071: left oid 6 (ObjNum 353 snap 93 seq_num 353) 2024-09-21T09:54:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-mon[31286]: pgmap v9: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 136 MiB/s rd, 50 MiB/s wr, 440 op/s 2024-09-21T09:54:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-mon[31286]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:54:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-mon[31286]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-21T09:54:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[39045]: pgmap v9: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 136 MiB/s rd, 50 MiB/s wr, 440 op/s 2024-09-21T09:54:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[39045]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:54:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[39045]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-21T09:54:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[96568]: pgmap v9: 121 pgs: 121 active+clean; 123 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 136 MiB/s rd, 50 MiB/s wr, 440 op/s 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[96568]: osdmap e257: 8 total, 8 up, 8 in 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[96568]: mgrmap e42: y(active, since 4s), standbys: x 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:35 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:35.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:36.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:36.455 INFO:tasks.rados.rados.0.smithi033.stdout:1068: done (3 left) 2024-09-21T09:54:36.455 INFO:tasks.rados.rados.0.smithi033.stdout:1070: done (2 left) 2024-09-21T09:54:36.455 INFO:tasks.rados.rados.0.smithi033.stdout:1071: done (1 left) 2024-09-21T09:54:36.455 INFO:tasks.rados.rados.0.smithi033.stdout:1072: done (0 left) 2024-09-21T09:54:36.455 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 94 2024-09-21T09:54:36.455 INFO:tasks.rados.rados.0.smithi033.stdout:1073: seq_num 354 ranges {2252800=737280,2990080=401408} 2024-09-21T09:54:36.471 INFO:tasks.rados.rados.0.smithi033.stdout:1073: writing smithi03385152-39 from 2252800 to 2990080 tid 1 2024-09-21T09:54:36.473 INFO:tasks.rados.rados.0.smithi033.stdout:1073: writing smithi03385152-39 from 2990080 to 3391488 tid 2 2024-09-21T09:54:36.473 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 94 2024-09-21T09:54:36.473 INFO:tasks.rados.rados.0.smithi033.stdout:1074: seq_num 355 ranges {0=753664,753664=401408} 2024-09-21T09:54:36.478 INFO:tasks.rados.rados.0.smithi033.stdout:1074: writing smithi03385152-43 from 0 to 753664 tid 1 2024-09-21T09:54:36.480 INFO:tasks.rados.rados.0.smithi033.stdout:1074: writing smithi03385152-43 from 753664 to 1155072 tid 2 2024-09-21T09:54:36.481 INFO:tasks.rados.rados.0.smithi033.stdout:1075: read oid 2 snap -1 2024-09-21T09:54:36.481 INFO:tasks.rados.rados.0.smithi033.stdout:1075: expect (ObjNum 297 snap 76 seq_num 297) 2024-09-21T09:54:36.481 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 94 2024-09-21T09:54:36.481 INFO:tasks.rados.rados.0.smithi033.stdout:1076: seq_num 356 ranges {3219456=770048,3989504=401408} 2024-09-21T09:54:36.503 INFO:tasks.rados.rados.0.smithi033.stdout:1076: writing smithi03385152-11 from 3219456 to 3989504 tid 1 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1076: writing smithi03385152-11 from 3989504 to 4390912 tid 2 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1077: rmattr oid 14 current snap is 94 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1073: finishing write tid 1 to smithi03385152-39 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1073: finishing write tid 2 to smithi03385152-39 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1073: finishing write tid 3 to smithi03385152-39 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1073: finishing write tid 4 to smithi03385152-39 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 128 (ObjNum 354 snap 94 seq_num 354) dirty exists 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1073: left oid 39 (ObjNum 354 snap 94 seq_num 354) 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1074: finishing write tid 1 to smithi03385152-43 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1074: finishing write tid 2 to smithi03385152-43 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1074: finishing write tid 3 to smithi03385152-43 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1073: done (4 left) 2024-09-21T09:54:36.506 INFO:tasks.rados.rados.0.smithi033.stdout:1078: rmattr oid 25 current snap is 94 2024-09-21T09:54:36.507 INFO:tasks.rados.rados.0.smithi033.stdout:1074: finishing write tid 4 to smithi03385152-43 2024-09-21T09:54:36.507 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 376 (ObjNum 355 snap 94 seq_num 355) dirty exists 2024-09-21T09:54:36.507 INFO:tasks.rados.rados.0.smithi033.stdout:1074: left oid 43 (ObjNum 355 snap 94 seq_num 355) 2024-09-21T09:54:36.507 INFO:tasks.rados.rados.0.smithi033.stdout:1075: expect (ObjNum 297 snap 76 seq_num 297) 2024-09-21T09:54:36.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 0 (ObjNum 257 snap 0 seq_num 1163894224) dirty dne 2024-09-21T09:54:36.571 INFO:tasks.rados.rados.0.smithi033.stdout:1076: finishing write tid 1 to smithi03385152-11 2024-09-21T09:54:36.571 INFO:tasks.rados.rados.0.smithi033.stdout:1076: finishing write tid 2 to smithi03385152-11 2024-09-21T09:54:36.572 INFO:tasks.rados.rados.0.smithi033.stdout:1074: done (4 left) 2024-09-21T09:54:36.572 INFO:tasks.rados.rados.0.smithi033.stdout:1075: done (3 left) 2024-09-21T09:54:36.572 INFO:tasks.rados.rados.0.smithi033.stdout:1078: done (2 left) 2024-09-21T09:54:36.572 INFO:tasks.rados.rados.0.smithi033.stdout:1079: snap_remove snap 92 2024-09-21T09:54:36.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[39045]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:54:36.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[96568]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:54:36.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:36 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:36 smithi105 ceph-mon[31286]: osdmap e258: 8 total, 8 up, 8 in 2024-09-21T09:54:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:36 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:36 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:36 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:36 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:37.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:37.458 INFO:tasks.rados.rados.0.smithi033.stdout:1079: done (2 left) 2024-09-21T09:54:37.458 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 94 2024-09-21T09:54:37.458 INFO:tasks.rados.rados.0.smithi033.stdout:1080: seq_num 357 ranges {1277952=786432,2064384=401408} 2024-09-21T09:54:37.469 INFO:tasks.rados.rados.0.smithi033.stdout:1080: writing smithi03385152-36 from 1277952 to 2064384 tid 1 2024-09-21T09:54:37.472 INFO:tasks.rados.rados.0.smithi033.stdout:1080: writing smithi03385152-36 from 2064384 to 2465792 tid 2 2024-09-21T09:54:37.472 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 94 2024-09-21T09:54:37.472 INFO:tasks.rados.rados.0.smithi033.stdout:1081: seq_num 358 ranges {0=802816,802816=401408} 2024-09-21T09:54:37.478 INFO:tasks.rados.rados.0.smithi033.stdout:1081: writing smithi03385152-37 from 0 to 802816 tid 1 2024-09-21T09:54:37.481 INFO:tasks.rados.rados.0.smithi033.stdout:1081: writing smithi03385152-37 from 802816 to 1204224 tid 2 2024-09-21T09:54:37.481 INFO:tasks.rados.rados.0.smithi033.stdout:1082: snap_create 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1076: finishing write tid 3 to smithi03385152-11 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 677 (ObjNum 276 snap 69 seq_num 276) dirty exists 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1076: finishing write tid 4 to smithi03385152-11 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 379 (ObjNum 356 snap 94 seq_num 356) dirty exists 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1076: left oid 11 (ObjNum 356 snap 94 seq_num 356) 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1080: finishing write tid 1 to smithi03385152-36 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1080: finishing write tid 2 to smithi03385152-36 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1080: finishing write tid 3 to smithi03385152-36 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1080: finishing write tid 4 to smithi03385152-36 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 677 (ObjNum 357 snap 94 seq_num 357) dirty exists 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1080: left oid 36 (ObjNum 357 snap 94 seq_num 357) 2024-09-21T09:54:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:1081: finishing write tid 1 to smithi03385152-37 2024-09-21T09:54:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:1081: finishing write tid 2 to smithi03385152-37 2024-09-21T09:54:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:1081: finishing write tid 3 to smithi03385152-37 2024-09-21T09:54:37.484 INFO:tasks.rados.rados.0.smithi033.stdout:1081: finishing write tid 4 to smithi03385152-37 2024-09-21T09:54:37.485 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 382 (ObjNum 358 snap 94 seq_num 358) dirty exists 2024-09-21T09:54:37.485 INFO:tasks.rados.rados.0.smithi033.stdout:1081: left oid 37 (ObjNum 358 snap 94 seq_num 358) 2024-09-21T09:54:37.657 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:37 smithi105 ceph-mon[31286]: pgmap v12: 121 pgs: 121 active+clean; 134 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-09-21T09:54:37.657 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:37 smithi105 ceph-mon[31286]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:54:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:37 smithi033 ceph-mon[39045]: pgmap v12: 121 pgs: 121 active+clean; 134 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-09-21T09:54:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:37 smithi033 ceph-mon[39045]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:54:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:37 smithi033 ceph-mon[96568]: pgmap v12: 121 pgs: 121 active+clean; 134 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-09-21T09:54:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:37 smithi033 ceph-mon[96568]: osdmap e259: 8 total, 8 up, 8 in 2024-09-21T09:54:37.893 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1076: done (4 left) 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1077: done (3 left) 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1080: done (2 left) 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1081: done (1 left) 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1082: done (0 left) 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1083: read oid 5 snap 89 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1083: expect (ObjNum 91534496 snap 21955 seq_num 91018896) 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 95 2024-09-21T09:54:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1084: seq_num 359 ranges {6004736=417792,6422528=417792,6840320=385024} 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (90s) 3s ago 5m 15.5M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (67s) 2s ago 5m 80.9M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (7m) 3s ago 7m 15.2M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (7m) 2s ago 7m 19.6M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (29s) 2s ago 10m 451M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (56s) 3s ago 12m 490M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (15s) 3s ago 12m 32.4M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (11m) 2s ago 11m 56.6M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c2741af3f853 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (11m) 3s ago 11m 58.2M 2048M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 0f8b68f40cc5 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (85s) 3s ago 5m 9805k - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (61s) 2s ago 5m 9.94M - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:54:38.485 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (10m) 3s ago 10m 151M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:54:38.486 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (10m) 3s ago 10m 156M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:54:38.486 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (9m) 3s ago 9m 139M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:54:38.486 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (9m) 3s ago 9m 92.5M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:54:38.486 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (9m) 2s ago 9m 156M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:54:38.486 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (8m) 2s ago 8m 172M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:54:38.486 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (8m) 2s ago 8m 107M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:54:38.486 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (7m) 2s ago 7m 125M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:54:38.486 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (34s) 2s ago 6m 43.7M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:54:38.508 INFO:tasks.rados.rados.0.smithi033.stdout:1084: writing smithi03385152-15 from 6004736 to 6422528 tid 1 2024-09-21T09:54:38.511 INFO:tasks.rados.rados.0.smithi033.stdout:1084: writing smithi03385152-15 from 6422528 to 6840320 tid 2 2024-09-21T09:54:38.513 INFO:tasks.rados.rados.0.smithi033.stdout:1084: writing smithi03385152-15 from 6840320 to 7225344 tid 3 2024-09-21T09:54:38.513 INFO:tasks.rados.rados.0.smithi033.stdout:1085: read oid 31 snap -1 2024-09-21T09:54:38.513 INFO:tasks.rados.rados.0.smithi033.stdout:1085: expect (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:54:38.513 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 95 2024-09-21T09:54:38.513 INFO:tasks.rados.rados.0.smithi033.stdout:1086: seq_num 360 ranges {3211264=434176,3645440=704512,4349952=98304} 2024-09-21T09:54:38.533 INFO:tasks.rados.rados.0.smithi033.stdout:1086: writing smithi03385152-23 from 3211264 to 3645440 tid 1 2024-09-21T09:54:38.537 INFO:tasks.rados.rados.0.smithi033.stdout:1086: writing smithi03385152-23 from 3645440 to 4349952 tid 2 2024-09-21T09:54:38.538 INFO:tasks.rados.rados.0.smithi033.stdout:1086: writing smithi03385152-23 from 4349952 to 4448256 tid 3 2024-09-21T09:54:38.538 INFO:tasks.rados.rados.0.smithi033.stdout:1087: snap_remove snap 94 2024-09-21T09:54:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:38 smithi033 ceph-mon[39045]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:54:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:38 smithi033 ceph-mon[39045]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:38.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:38 smithi033 ceph-mon[96568]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:54:38.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:38 smithi033 ceph-mon[96568]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:38 smithi105 ceph-mon[31286]: osdmap e260: 8 total, 8 up, 8 in 2024-09-21T09:54:38.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:38 smithi105 ceph-mon[31286]: from='client.34146 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2, 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:54:38.845 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 12, 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:54:38.846 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:54:39.145 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:54:39.145 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:54:39.145 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:54:39.145 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:54:39.146 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:54:39.146 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:54:39.146 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:54:39.146 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "3/20 daemons upgraded", 2024-09-21T09:54:39.146 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "", 2024-09-21T09:54:39.146 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:54:39.146 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:54:39.465 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:39.465 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: pgmap v15: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.4 MiB/s wr, 46 op/s 2024-09-21T09:54:39.465 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:54:39.465 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: from='client.25205 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:39.474 INFO:tasks.rados.rados.0.smithi033.stdout:1087: done (4 left) 2024-09-21T09:54:39.475 INFO:tasks.rados.rados.0.smithi033.stdout:1088: snap_create 2024-09-21T09:54:39.475 INFO:tasks.rados.rados.0.smithi033.stdout:1083: expect (ObjNum 91534496 snap 21955 seq_num 91018896) 2024-09-21T09:54:39.478 INFO:tasks.rados.rados.0.smithi033.stdout:1084: finishing write tid 1 to smithi03385152-15 2024-09-21T09:54:39.478 INFO:tasks.rados.rados.0.smithi033.stdout:1084: finishing write tid 2 to smithi03385152-15 2024-09-21T09:54:39.478 INFO:tasks.rados.rados.0.smithi033.stdout:1084: finishing write tid 3 to smithi03385152-15 2024-09-21T09:54:39.478 INFO:tasks.rados.rados.0.smithi033.stdout:1084: finishing write tid 4 to smithi03385152-15 2024-09-21T09:54:39.479 INFO:tasks.rados.rados.0.smithi033.stdout:1085: expect (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:54:39.504 INFO:tasks.rados.rados.0.smithi033.stdout:1084: finishing write tid 5 to smithi03385152-15 2024-09-21T09:54:39.504 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 330 (ObjNum 359 snap 95 seq_num 359) dirty exists 2024-09-21T09:54:39.504 INFO:tasks.rados.rados.0.smithi033.stdout:1084: left oid 15 (ObjNum 359 snap 95 seq_num 359) 2024-09-21T09:54:39.504 INFO:tasks.rados.rados.0.smithi033.stdout:1086: finishing write tid 1 to smithi03385152-23 2024-09-21T09:54:39.504 INFO:tasks.rados.rados.0.smithi033.stdout:1086: finishing write tid 2 to smithi03385152-23 2024-09-21T09:54:39.504 INFO:tasks.rados.rados.0.smithi033.stdout:1086: finishing write tid 3 to smithi03385152-23 2024-09-21T09:54:39.504 INFO:tasks.rados.rados.0.smithi033.stdout:1086: finishing write tid 4 to smithi03385152-23 2024-09-21T09:54:39.504 INFO:tasks.rados.rados.0.smithi033.stdout:1086: finishing write tid 5 to smithi03385152-23 2024-09-21T09:54:39.505 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 364 (ObjNum 360 snap 95 seq_num 360) dirty exists 2024-09-21T09:54:39.505 INFO:tasks.rados.rados.0.smithi033.stdout:1086: left oid 23 (ObjNum 360 snap 95 seq_num 360) 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: pgmap v15: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.4 MiB/s wr, 46 op/s 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: from='client.25205 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: from='client.? 172.21.15.33:0/2591092028' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:39.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:39 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2591092028' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: pgmap v15: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 5.4 MiB/s wr, 46 op/s 2024-09-21T09:54:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: osdmap e261: 8 total, 8 up, 8 in 2024-09-21T09:54:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: from='client.25205 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: from='client.? 172.21.15.33:0/2591092028' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:39 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:40.471 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:40.483 INFO:tasks.rados.rados.0.smithi033.stdout:1083: done (4 left) 2024-09-21T09:54:40.483 INFO:tasks.rados.rados.0.smithi033.stdout:1084: done (3 left) 2024-09-21T09:54:40.483 INFO:tasks.rados.rados.0.smithi033.stdout:1085: done (2 left) 2024-09-21T09:54:40.483 INFO:tasks.rados.rados.0.smithi033.stdout:1086: done (1 left) 2024-09-21T09:54:40.483 INFO:tasks.rados.rados.0.smithi033.stdout:1088: done (0 left) 2024-09-21T09:54:40.483 INFO:tasks.rados.rados.0.smithi033.stdout:1089: rollback oid 2 current snap is 96 2024-09-21T09:54:40.483 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 95 2024-09-21T09:54:40.483 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 96 2024-09-21T09:54:40.484 INFO:tasks.rados.rados.0.smithi033.stdout:1090: seq_num 361 ranges {0=450560,450560=581632,1032192=221184} 2024-09-21T09:54:40.485 INFO:tasks.rados.rados.0.smithi033.stdout:1090: writing smithi03385152-38 from 0 to 450560 tid 1 2024-09-21T09:54:40.489 INFO:tasks.rados.rados.0.smithi033.stdout:1090: writing smithi03385152-38 from 450560 to 1032192 tid 2 2024-09-21T09:54:40.491 INFO:tasks.rados.rados.0.smithi033.stdout:1090: writing smithi03385152-38 from 1032192 to 1253376 tid 3 2024-09-21T09:54:40.491 INFO:tasks.rados.rados.0.smithi033.stdout:1091: read oid 40 snap -1 2024-09-21T09:54:40.491 INFO:tasks.rados.rados.0.smithi033.stdout:1091: expect (ObjNum 114 snap 25 seq_num 114) 2024-09-21T09:54:40.491 INFO:tasks.rados.rados.0.smithi033.stdout:1092: snap_remove snap 91 2024-09-21T09:54:40.569 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:40.570 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T09:54:40.571 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.572 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.573 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.573 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:40.573 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:40.573 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:40.573 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:40 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T09:54:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='client.34170 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:54:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.conf 2024-09-21T09:54:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.conf 2024-09-21T09:54:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.conf 2024-09-21T09:54:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: osdmap e262: 8 total, 8 up, 8 in 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: Updating smithi105:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: Updating smithi033:/etc/ceph/ceph.client.admin.keyring 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: Updating smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: Updating smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/config/ceph.client.admin.keyring 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:40.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:40 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-21T09:54:41.491 INFO:tasks.rados.rados.0.smithi033.stdout:1092: done (3 left) 2024-09-21T09:54:41.492 INFO:tasks.rados.rados.0.smithi033.stdout:1093: read oid 5 snap -1 2024-09-21T09:54:41.492 INFO:tasks.rados.rados.0.smithi033.stdout:1093: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:54:41.492 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 96 2024-09-21T09:54:41.492 INFO:tasks.rados.rados.0.smithi033.stdout:1094: seq_num 362 ranges {1794048=466944,2260992=466944,2727936=335872} 2024-09-21T09:54:41.504 INFO:tasks.rados.rados.0.smithi033.stdout:1094: writing smithi03385152-16 from 1794048 to 2260992 tid 1 2024-09-21T09:54:41.507 INFO:tasks.rados.rados.0.smithi033.stdout:1094: writing smithi03385152-16 from 2260992 to 2727936 tid 2 2024-09-21T09:54:41.509 INFO:tasks.rados.rados.0.smithi033.stdout:1094: writing smithi03385152-16 from 2727936 to 3063808 tid 3 2024-09-21T09:54:41.509 INFO:tasks.rados.rados.0.smithi033.stdout:1095: delete oid 29 current snap is 96 2024-09-21T09:54:41.509 INFO:tasks.rados.rados.0.smithi033.stdout:1089: finishing rollback tid 0 to smithi03385152-2 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1089: finishing rollback tid 1 to smithi03385152-2 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1089: finishing rollback tid 2 to smithi03385152-2 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 606 (ObjNum 297 snap 76 seq_num 297) dirty exists 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1090: finishing write tid 1 to smithi03385152-38 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1090: finishing write tid 2 to smithi03385152-38 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1090: finishing write tid 3 to smithi03385152-38 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1090: finishing write tid 4 to smithi03385152-38 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1090: finishing write tid 5 to smithi03385152-38 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 624 (ObjNum 361 snap 96 seq_num 361) dirty exists 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1090: left oid 38 (ObjNum 361 snap 96 seq_num 361) 2024-09-21T09:54:41.510 INFO:tasks.rados.rados.0.smithi033.stdout:1091: expect (ObjNum 114 snap 25 seq_num 114) 2024-09-21T09:54:41.594 INFO:tasks.rados.rados.0.smithi033.stdout:1093: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:54:41.599 INFO:tasks.rados.rados.0.smithi033.stdout:1094: finishing write tid 1 to smithi03385152-16 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1094: finishing write tid 2 to smithi03385152-16 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1094: finishing write tid 3 to smithi03385152-16 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1089: done (5 left) 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1090: done (4 left) 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1091: done (3 left) 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1093: done (2 left) 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1095: done (1 left) 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1096: read oid 30 snap -1 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1096: expect (ObjNum 289 snap 70 seq_num 289) 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1097: read oid 22 snap -1 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1097: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1098: delete oid 39 current snap is 96 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1094: finishing write tid 4 to smithi03385152-16 2024-09-21T09:54:41.600 INFO:tasks.rados.rados.0.smithi033.stdout:1094: finishing write tid 5 to smithi03385152-16 2024-09-21T09:54:41.601 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 684 (ObjNum 362 snap 96 seq_num 362) dirty exists 2024-09-21T09:54:41.601 INFO:tasks.rados.rados.0.smithi033.stdout:1094: left oid 16 (ObjNum 362 snap 96 seq_num 362) 2024-09-21T09:54:41.601 INFO:tasks.rados.rados.0.smithi033.stdout:1094: done (3 left) 2024-09-21T09:54:41.601 INFO:tasks.rados.rados.0.smithi033.stdout:1098: done (2 left) 2024-09-21T09:54:41.601 INFO:tasks.rados.rados.0.smithi033.stdout:1099: snap_remove snap 89 2024-09-21T09:54:41.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:54:41] "GET /metrics HTTP/1.1" 200 33458 "" "Prometheus/2.51.0" 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: pgmap v18: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 1 op/s 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[39045]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:54:41.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: pgmap v18: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 1 op/s 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:41.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:41.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:54:41.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:54:41.752 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: Upgrade: It appears safe to stop mon.c 2024-09-21T09:54:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: pgmap v18: 121 pgs: 121 active+clean; 143 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 2.2 MiB/s wr, 1 op/s 2024-09-21T09:54:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: osdmap e263: 8 total, 8 up, 8 in 2024-09-21T09:54:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config-key dump", "format": "json"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd='heap' args=[stats]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='admin socket' entity='admin socket' cmd=heap args=[stats]: finished 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:41.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:54:41.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:54:41.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:41 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:42.510 INFO:tasks.rados.rados.0.smithi033.stdout:1099: done (2 left) 2024-09-21T09:54:42.511 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 96 2024-09-21T09:54:42.511 INFO:tasks.rados.rados.0.smithi033.stdout:1100: seq_num 365 ranges {0=516096,516096=573440,1089536=229376} 2024-09-21T09:54:42.514 INFO:tasks.rados.rados.0.smithi033.stdout:1100: writing smithi03385152-12 from 0 to 516096 tid 1 2024-09-21T09:54:42.519 INFO:tasks.rados.rados.0.smithi033.stdout:1100: writing smithi03385152-12 from 516096 to 1089536 tid 2 2024-09-21T09:54:42.520 INFO:tasks.rados.rados.0.smithi033.stdout:1100: writing smithi03385152-12 from 1089536 to 1318912 tid 3 2024-09-21T09:54:42.520 INFO:tasks.rados.rados.0.smithi033.stdout:1096: expect (ObjNum 289 snap 70 seq_num 289) 2024-09-21T09:54:42.689 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:42 smithi033 ceph-mon[96568]: Upgrade: Updating mon.c 2024-09-21T09:54:42.689 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:42 smithi033 ceph-mon[96568]: Deploying daemon mon.c on smithi033 2024-09-21T09:54:42.689 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:42 smithi033 ceph-mon[96568]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:54:42.689 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:42 smithi033 ceph-mon[39045]: Upgrade: Updating mon.c 2024-09-21T09:54:42.689 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:42 smithi033 ceph-mon[39045]: Deploying daemon mon.c on smithi033 2024-09-21T09:54:42.689 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:42 smithi033 ceph-mon[39045]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:54:42.701 INFO:tasks.rados.rados.0.smithi033.stdout:1097: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:54:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:42 smithi105 ceph-mon[31286]: Upgrade: Updating mon.c 2024-09-21T09:54:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:42 smithi105 ceph-mon[31286]: Deploying daemon mon.c on smithi033 2024-09-21T09:54:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:42 smithi105 ceph-mon[31286]: osdmap e264: 8 total, 8 up, 8 in 2024-09-21T09:54:42.977 INFO:tasks.rados.rados.0.smithi033.stdout:1100: finishing write tid 1 to smithi03385152-12 2024-09-21T09:54:42.977 INFO:tasks.rados.rados.0.smithi033.stdout:1100: finishing write tid 2 to smithi03385152-12 2024-09-21T09:54:42.977 INFO:tasks.rados.rados.0.smithi033.stdout:1100: finishing write tid 3 to smithi03385152-12 2024-09-21T09:54:42.977 INFO:tasks.rados.rados.0.smithi033.stdout:1100: finishing write tid 4 to smithi03385152-12 2024-09-21T09:54:42.977 INFO:tasks.rados.rados.0.smithi033.stdout:1096: done (2 left) 2024-09-21T09:54:42.977 INFO:tasks.rados.rados.0.smithi033.stdout:1097: done (1 left) 2024-09-21T09:54:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 96 2024-09-21T09:54:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:1100: finishing write tid 5 to smithi03385152-12 2024-09-21T09:54:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 439 (ObjNum 365 snap 96 seq_num 365) dirty exists 2024-09-21T09:54:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:1100: left oid 12 (ObjNum 365 snap 96 seq_num 365) 2024-09-21T09:54:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:1101: seq_num 366 ranges {3563520=532480,4096000=458752,4554752=344064} 2024-09-21T09:54:42.999 INFO:tasks.rados.rados.0.smithi033.stdout:1101: writing smithi03385152-30 from 3563520 to 4096000 tid 1 2024-09-21T09:54:43.002 INFO:tasks.rados.rados.0.smithi033.stdout:1101: writing smithi03385152-30 from 4096000 to 4554752 tid 2 2024-09-21T09:54:43.004 INFO:tasks.rados.rados.0.smithi033.stdout:1101: writing smithi03385152-30 from 4554752 to 4898816 tid 3 2024-09-21T09:54:43.004 INFO:tasks.rados.rados.0.smithi033.stdout:1100: done (1 left) 2024-09-21T09:54:43.005 INFO:tasks.rados.rados.0.smithi033.stdout:1102: copy_from oid 15 from oid 38 current snap is 96 2024-09-21T09:54:43.005 INFO:tasks.rados.rados.0.smithi033.stdout:1101: finishing write tid 1 to smithi03385152-30 2024-09-21T09:54:43.005 INFO:tasks.rados.rados.0.smithi033.stdout:1103: snap_remove snap 93 2024-09-21T09:54:43.237 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:42 smithi033 systemd[1]: Stopping Ceph mon.c for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:43.492 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:43] ENGINE Bus STOPPING 2024-09-21T09:54:43.493 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c[39022]: 2024-09-21T09:54:43.234+0000 7faede666640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:54:43.493 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c[39022]: 2024-09-21T09:54:43.234+0000 7faede666640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:54:43.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:43 smithi033 podman[98345]: 2024-09-21 09:54:43.491296286 +0000 UTC m=+0.366323865 container died 0f8b68f40cc5bb185d5cf2e97bed9b851a36e8ab5b6f4f952bc425bcb70e97f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T09:54:43.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:54:43.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:43] ENGINE Bus STOPPED 2024-09-21T09:54:44.170 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:43 smithi033 podman[98345]: 2024-09-21 09:54:43.958798284 +0000 UTC m=+0.833825855 container cleanup 0f8b68f40cc5bb185d5cf2e97bed9b851a36e8ab5b6f4f952bc425bcb70e97f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T09:54:44.170 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:43 smithi033 bash[98345]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c 2024-09-21T09:54:44.425 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:44 smithi033 podman[98362]: 2024-09-21 09:54:44.168700497 +0000 UTC m=+0.674072772 container remove 0f8b68f40cc5bb185d5cf2e97bed9b851a36e8ab5b6f4f952bc425bcb70e97f2 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T09:54:44.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:44.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:44 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.c.service: Deactivated successfully. 2024-09-21T09:54:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:44 smithi033 systemd[1]: Stopped Ceph mon.c for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:44 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.c.service: Consumed 8.558s CPU time. 2024-09-21T09:54:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:44 smithi033 systemd[1]: Starting Ceph mon.c for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:54:45.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:44 smithi033 podman[98507]: 2024-09-21 09:54:44.86123779 +0000 UTC m=+0.103179478 container create 5c9071ecac424d4a7b79d07a266b324d27ab517bf48bbf2c07d0cd2cedf4ae18 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-21T09:54:45.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:44 smithi033 podman[98507]: 2024-09-21 09:54:44.775127029 +0000 UTC m=+0.017068711 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:54:45.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 podman[98507]: 2024-09-21 09:54:45.063822986 +0000 UTC m=+0.305764667 container init 5c9071ecac424d4a7b79d07a266b324d27ab517bf48bbf2c07d0cd2cedf4ae18 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:54:45.121 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 podman[98507]: 2024-09-21 09:54:45.06825955 +0000 UTC m=+0.310201234 container start 5c9071ecac424d4a7b79d07a266b324d27ab517bf48bbf2c07d0cd2cedf4ae18 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: pidfile_write: ignore empty --pid-file 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: load: jerasure load: lrc 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Git sha 0 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: DB SUMMARY 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: DB Session ID: CRKFJVBBGVZJI12ACS0N 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1096 Bytes 2024-09-21T09:54:45.122 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 11415613 ; 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.env: 0x56262778c120 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.info_log: 0x562629b4d700 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.statistics: (nil) 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.use_fsync: 0 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:54:45.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.db_log_dir: 2024-09-21T09:54:45.124 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.wal_dir: 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.write_buffer_manager: 0x562629b501e0 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:54:45.125 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:54:45.126 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:54:45.126 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.unordered_write: 0 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.row_cache: None 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.wal_filter: None 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:54:45.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.wal_compression: 0 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:54:45.128 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_open_files: -1 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:54:45.129 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Compression algorithms supported: 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: kZSTD supported: 0 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:54:45.130 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.merge_operator: 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_filter: None 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562629b4d320) 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_type: 0 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: data_block_index_type: 0 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_shortening: 1 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: checksum: 4 2024-09-21T09:54:45.131 INFO:journalctl@ceph.mon.c.smithi033.stdout: no_block_cache: 0 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache: 0x562629b41610 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_options: 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: capacity : 536870912 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: num_shard_bits : 4 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: strict_capacity_limit : 0 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_cache_compressed: (nil) 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: persistent_cache: (nil) 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_size: 4096 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_size_deviation: 10 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_restart_interval: 16 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: index_block_restart_interval: 1 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: metadata_block_size: 4096 2024-09-21T09:54:45.132 INFO:journalctl@ceph.mon.c.smithi033.stdout: partition_filters: 0 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: use_delta_encoding: 1 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: filter_policy: bloomfilter 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: whole_key_filtering: 1 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: verify_compression: 0 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: format_version: 5 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: enable_index_compression: 1 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: block_align: 0 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: max_auto_readahead_size: 262144 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: prepopulate_block_cache: 0 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression: NoCompression 2024-09-21T09:54:45.133 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.num_levels: 7 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:54:45.134 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:54:45.135 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:54:45.136 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:54:45.137 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.ttl: 2592000 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:54:45.138 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13415, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8c4b7523-fe02-42b8-a674-b369bcc4a640 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912485095583, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:54:45.139 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912485119409, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7098068, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13412, "largest_seqno": 17025, "table_properties": {"data_size": 7081700, "index_size": 11289, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4037, "raw_key_size": 36640, "raw_average_key_size": 22, "raw_value_size": 7048090, "raw_average_value_size": 4407, "num_data_blocks": 489, "num_entries": 1599, "num_filter_entries": 1599, "num_deletions": 310, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726912485, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8c4b7523-fe02-42b8-a674-b369bcc4a640", "db_session_id": "CRKFJVBBGVZJI12ACS0N", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:54:45.140 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912485119539, "job": 1, "event": "recovery_finished"} 2024-09-21T09:54:45.140 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:54:45.140 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:54:45.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:54:45.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562629b70e00 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: DB pointer 0x562629c72000 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** DB Stats ** 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:54:45.497 INFO:journalctl@ceph.mon.c.smithi033.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: L0 1/0 6.77 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 781.2 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: L6 1/0 7.64 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: Sum 2/0 14.41 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 781.2 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 781.2 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** Compaction Stats [default] ** 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 781.2 0.01 0.00 1 0.009 0 0 0.0 0.0 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:54:45.498 INFO:journalctl@ceph.mon.c.smithi033.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: Cumulative compaction: 0.01 GB write, 203.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: Interval compaction: 0.01 GB write, 203.94 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: Block cache BinnedLRUCache@0x562629b41610#7 capacity: 512.00 MB usage: 16.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.03 KB,0.0007689%) IndexBlock(1,12.08 KB,0.00230372%) Misc(1,0.00 KB,0%) 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: starting mon.c rank 2 at public addrs [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] at bind addrs [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: mon.c@-1(???) e3 preinit fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: mon.c@-1(???).mds e7 new map 2024-09-21T09:54:45.499 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: mon.c@-1(???).mds e7 print_map 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: e7 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: legacy client fscid: 1 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: Filesystem 'cephfs' (1) 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: fs_name cephfs 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: epoch 7 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: created 2024-09-21T09:47:25.959476+0000 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: modified 2024-09-21T09:47:39.447745+0000 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: tableserver 0 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: root 0 2024-09-21T09:54:45.500 INFO:journalctl@ceph.mon.c.smithi033.stdout: session_timeout 60 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: session_autoclose 300 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: max_file_size 1099511627776 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: max_xattr_size 65536 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: required_client_features {} 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: last_failure 0 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: last_failure_osd_epoch 0 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: max_mds 1 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: in 0 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: up {0=24358} 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: failed 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: damaged 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: stopped 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: data_pools [3] 2024-09-21T09:54:45.501 INFO:journalctl@ceph.mon.c.smithi033.stdout: metadata_pool 2 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: inline_data disabled 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: balancer 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: bal_rank_mask -1 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: standby_count_wanted 0 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: [mds.b{0:24358} state up:active seq 5 addr [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: Standby daemons: 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout: [mds.a{-1:24323} state up:standby seq 1 addr [v2:172.21.15.33:6834/125068843,v1:172.21.15.33:6835/125068843] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: mon.c@-1(???).osd e265 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: mon.c@-1(???).osd e265 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: mon.c@-1(???).osd e265 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:54:45.502 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: mon.c@-1(???).osd e265 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:54:45.503 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 ceph-mon[98546]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-21T09:54:45.503 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 bash[98507]: 5c9071ecac424d4a7b79d07a266b324d27ab517bf48bbf2c07d0cd2cedf4ae18 2024-09-21T09:54:45.503 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:45 smithi033 systemd[1]: Started Ceph mon.c for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:54:46.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:46.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:47.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:50.364 INFO:tasks.rados.rados.0.smithi033.stdout:1103: done (2 left) 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1104: read oid 27 snap 95 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1104: expect (ObjNum 349 snap 92 seq_num 349) 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1105: read oid 37 snap 95 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1105: expect (ObjNum 358 snap 94 seq_num 358) 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1106: snap_create 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1101: finishing write tid 2 to smithi03385152-30 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1101: finishing write tid 3 to smithi03385152-30 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1101: finishing write tid 4 to smithi03385152-30 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1101: finishing write tid 5 to smithi03385152-30 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 699 (ObjNum 366 snap 96 seq_num 366) dirty exists 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1101: left oid 30 (ObjNum 366 snap 96 seq_num 366) 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1102: finishing copy_from racing read to smithi03385152-15 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:1102: finishing copy_from to smithi03385152-15 2024-09-21T09:54:50.365 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 624 (ObjNum 361 snap 96 seq_num 361) dirty exists 2024-09-21T09:54:50.373 INFO:tasks.rados.rados.0.smithi033.stdout:1105: expect (ObjNum 358 snap 94 seq_num 358) 2024-09-21T09:54:50.409 INFO:tasks.rados.rados.0.smithi033.stdout:1104: expect (ObjNum 349 snap 92 seq_num 349) 2024-09-21T09:54:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:50.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:50] ENGINE Bus STARTING 2024-09-21T09:54:50.747 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:50] ENGINE Serving on http://:::9283 2024-09-21T09:54:50.747 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:54:50] ENGINE Bus STARTED 2024-09-21T09:54:51.390 INFO:tasks.rados.rados.0.smithi033.stdout:1101: done (4 left) 2024-09-21T09:54:51.390 INFO:tasks.rados.rados.0.smithi033.stdout:1102: done (3 left) 2024-09-21T09:54:51.391 INFO:tasks.rados.rados.0.smithi033.stdout:1104: done (2 left) 2024-09-21T09:54:51.391 INFO:tasks.rados.rados.0.smithi033.stdout:1105: done (1 left) 2024-09-21T09:54:51.391 INFO:tasks.rados.rados.0.smithi033.stdout:1106: done (0 left) 2024-09-21T09:54:51.391 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 97 2024-09-21T09:54:51.391 INFO:tasks.rados.rados.0.smithi033.stdout:1107: seq_num 367 ranges {9191424=548864,9740288=737280,10477568=65536} 2024-09-21T09:54:51.443 INFO:tasks.rados.rados.0.smithi033.stdout:1107: writing smithi03385152-14 from 9191424 to 9740288 tid 1 2024-09-21T09:54:51.447 INFO:tasks.rados.rados.0.smithi033.stdout:1107: writing smithi03385152-14 from 9740288 to 10477568 tid 2 2024-09-21T09:54:51.448 INFO:tasks.rados.rados.0.smithi033.stdout:1107: writing smithi03385152-14 from 10477568 to 10543104 tid 3 2024-09-21T09:54:51.448 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 97 2024-09-21T09:54:51.448 INFO:tasks.rados.rados.0.smithi033.stdout:1108: seq_num 368 ranges {0=573440,573440=622592,1196032=180224} 2024-09-21T09:54:51.451 INFO:tasks.rados.rados.0.smithi033.stdout:1108: writing smithi03385152-39 from 0 to 573440 tid 1 2024-09-21T09:54:51.455 INFO:tasks.rados.rados.0.smithi033.stdout:1108: writing smithi03385152-39 from 573440 to 1196032 tid 2 2024-09-21T09:54:51.456 INFO:tasks.rados.rados.0.smithi033.stdout:1108: writing smithi03385152-39 from 1196032 to 1376256 tid 3 2024-09-21T09:54:51.456 INFO:tasks.rados.rados.0.smithi033.stdout:1109: read oid 9 snap -1 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:1109: expect (ObjNum 351 snap 92 seq_num 351) 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 97 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:1107: finishing write tid 1 to smithi03385152-14 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:1107: finishing write tid 2 to smithi03385152-14 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:1107: finishing write tid 3 to smithi03385152-14 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:1107: finishing write tid 4 to smithi03385152-14 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:1107: finishing write tid 5 to smithi03385152-14 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 681 (ObjNum 367 snap 97 seq_num 367) dirty exists 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:1107: left oid 14 (ObjNum 367 snap 97 seq_num 367) 2024-09-21T09:54:51.457 INFO:tasks.rados.rados.0.smithi033.stdout:1110: seq_num 369 ranges {5152768=589824,5742592=507904,6250496=294912} 2024-09-21T09:54:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:1110: writing smithi03385152-28 from 5152768 to 5742592 tid 1 2024-09-21T09:54:51.491 INFO:tasks.rados.rados.0.smithi033.stdout:1110: writing smithi03385152-28 from 5742592 to 6250496 tid 2 2024-09-21T09:54:51.493 INFO:tasks.rados.rados.0.smithi033.stdout:1110: writing smithi03385152-28 from 6250496 to 6545408 tid 3 2024-09-21T09:54:51.493 INFO:tasks.rados.rados.0.smithi033.stdout:1107: done (3 left) 2024-09-21T09:54:51.493 INFO:tasks.rados.rados.0.smithi033.stdout:1111: snap_create 2024-09-21T09:54:51.493 INFO:tasks.rados.rados.0.smithi033.stdout:1108: finishing write tid 1 to smithi03385152-39 2024-09-21T09:54:51.493 INFO:tasks.rados.rados.0.smithi033.stdout:1108: finishing write tid 2 to smithi03385152-39 2024-09-21T09:54:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:1108: finishing write tid 3 to smithi03385152-39 2024-09-21T09:54:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:1108: finishing write tid 4 to smithi03385152-39 2024-09-21T09:54:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:1108: finishing write tid 5 to smithi03385152-39 2024-09-21T09:54:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 144 (ObjNum 368 snap 97 seq_num 368) dirty exists 2024-09-21T09:54:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:1108: left oid 39 (ObjNum 368 snap 97 seq_num 368) 2024-09-21T09:54:51.494 INFO:tasks.rados.rados.0.smithi033.stdout:1109: expect (ObjNum 351 snap 92 seq_num 351) 2024-09-21T09:54:51.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: mon.c calling monitor election 2024-09-21T09:54:51.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: mon.b calling monitor election 2024-09-21T09:54:51.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: mon.a calling monitor election 2024-09-21T09:54:51.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:54:51.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: monmap epoch 3 2024-09-21T09:54:51.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:51.621 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: last_changed 2024-09-21T09:43:29.505849+0000 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: min_mon_release 17 (quincy) 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: election_strategy: 1 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: mgrmap e43: y(active, since 21s), standbys: x 2024-09-21T09:54:51.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[98546]: overall HEALTH_OK 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: mon.c calling monitor election 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: mon.b calling monitor election 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: mon.a calling monitor election 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: monmap epoch 3 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: last_changed 2024-09-21T09:43:29.505849+0000 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: min_mon_release 17 (quincy) 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: election_strategy: 1 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:54:51.623 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:54:51.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: mgrmap e43: y(active, since 21s), standbys: x 2024-09-21T09:54:51.624 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-mon[96568]: overall HEALTH_OK 2024-09-21T09:54:51.658 INFO:tasks.rados.rados.0.smithi033.stdout:1110: finishing write tid 1 to smithi03385152-28 2024-09-21T09:54:51.658 INFO:tasks.rados.rados.0.smithi033.stdout:1110: finishing write tid 2 to smithi03385152-28 2024-09-21T09:54:51.658 INFO:tasks.rados.rados.0.smithi033.stdout:1110: finishing write tid 3 to smithi03385152-28 2024-09-21T09:54:51.658 INFO:tasks.rados.rados.0.smithi033.stdout:1110: finishing write tid 4 to smithi03385152-28 2024-09-21T09:54:51.658 INFO:tasks.rados.rados.0.smithi033.stdout:1110: finishing write tid 5 to smithi03385152-28 2024-09-21T09:54:51.659 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 151 (ObjNum 369 snap 97 seq_num 369) dirty exists 2024-09-21T09:54:51.659 INFO:tasks.rados.rados.0.smithi033.stdout:1110: left oid 28 (ObjNum 369 snap 97 seq_num 369) 2024-09-21T09:54:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: mon.c calling monitor election 2024-09-21T09:54:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: mon.b calling monitor election 2024-09-21T09:54:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: mon.a calling monitor election 2024-09-21T09:54:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:54:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: monmap epoch 3 2024-09-21T09:54:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: last_changed 2024-09-21T09:43:29.505849+0000 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: min_mon_release 17 (quincy) 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: election_strategy: 1 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: osdmap e266: 8 total, 8 up, 8 in 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: mgrmap e43: y(active, since 21s), standbys: x 2024-09-21T09:54:51.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:51 smithi105 ceph-mon[31286]: overall HEALTH_OK 2024-09-21T09:54:51.946 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:54:51] "GET /metrics HTTP/1.1" 200 36205 "" "Prometheus/2.51.0" 2024-09-21T09:54:52.404 INFO:tasks.rados.rados.0.smithi033.stdout:1108: done (3 left) 2024-09-21T09:54:52.404 INFO:tasks.rados.rados.0.smithi033.stdout:1109: done (2 left) 2024-09-21T09:54:52.404 INFO:tasks.rados.rados.0.smithi033.stdout:1110: done (1 left) 2024-09-21T09:54:52.405 INFO:tasks.rados.rados.0.smithi033.stdout:1111: done (0 left) 2024-09-21T09:54:52.405 INFO:tasks.rados.rados.0.smithi033.stdout:1112: snap_create 2024-09-21T09:54:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:52 smithi033 ceph-mon[98546]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:54:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:52 smithi033 ceph-mon[96568]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:54:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:52 smithi105 ceph-mon[31286]: osdmap e267: 8 total, 8 up, 8 in 2024-09-21T09:54:53.405 INFO:tasks.rados.rados.0.smithi033.stdout:1112: done (0 left) 2024-09-21T09:54:53.406 INFO:tasks.rados.rados.0.smithi033.stdout:1113: delete oid 18 current snap is 99 2024-09-21T09:54:53.408 INFO:tasks.rados.rados.0.smithi033.stdout:1113: done (0 left) 2024-09-21T09:54:53.409 INFO:tasks.rados.rados.0.smithi033.stdout:1114: read oid 32 snap -1 2024-09-21T09:54:53.409 INFO:tasks.rados.rados.0.smithi033.stdout:1114: expect (ObjNum 334 snap 87 seq_num 334) 2024-09-21T09:54:53.409 INFO:tasks.rados.rados.0.smithi033.stdout:1115: delete oid 39 current snap is 99 2024-09-21T09:54:53.414 INFO:tasks.rados.rados.0.smithi033.stdout:1115: done (1 left) 2024-09-21T09:54:53.415 INFO:tasks.rados.rados.0.smithi033.stdout:1116: delete oid 37 current snap is 99 2024-09-21T09:54:53.418 INFO:tasks.rados.rados.0.smithi033.stdout:1116: done (1 left) 2024-09-21T09:54:53.418 INFO:tasks.rados.rados.0.smithi033.stdout:1117: rollback oid 2 current snap is 99 2024-09-21T09:54:53.418 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 96 2024-09-21T09:54:53.418 INFO:tasks.rados.rados.0.smithi033.stdout:1118: setattr oid 36 current snap is 99 2024-09-21T09:54:53.421 INFO:tasks.rados.rados.0.smithi033.stdout:1117: finishing rollback tid 0 to smithi03385152-2 2024-09-21T09:54:53.421 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 99 2024-09-21T09:54:53.421 INFO:tasks.rados.rados.0.smithi033.stdout:1119: seq_num 373 ranges {4448256=655360,5103616=499712,5603328=303104} 2024-09-21T09:54:53.449 INFO:tasks.rados.rados.0.smithi033.stdout:1119: writing smithi03385152-23 from 4448256 to 5103616 tid 1 2024-09-21T09:54:53.453 INFO:tasks.rados.rados.0.smithi033.stdout:1119: writing smithi03385152-23 from 5103616 to 5603328 tid 2 2024-09-21T09:54:53.455 INFO:tasks.rados.rados.0.smithi033.stdout:1119: writing smithi03385152-23 from 5603328 to 5906432 tid 3 2024-09-21T09:54:53.455 INFO:tasks.rados.rados.0.smithi033.stdout:1120: snap_create 2024-09-21T09:54:53.455 INFO:tasks.rados.rados.0.smithi033.stdout:1117: finishing rollback tid 1 to smithi03385152-2 2024-09-21T09:54:53.455 INFO:tasks.rados.rados.0.smithi033.stdout:1117: finishing rollback tid 2 to smithi03385152-2 2024-09-21T09:54:53.455 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 609 (ObjNum 297 snap 76 seq_num 297) dirty exists 2024-09-21T09:54:53.455 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 709 (ObjNum 357 snap 94 seq_num 357) dirty exists 2024-09-21T09:54:53.455 INFO:tasks.rados.rados.0.smithi033.stdout:1114: expect (ObjNum 334 snap 87 seq_num 334) 2024-09-21T09:54:53.662 INFO:tasks.rados.rados.0.smithi033.stdout:1119: finishing write tid 1 to smithi03385152-23 2024-09-21T09:54:53.662 INFO:tasks.rados.rados.0.smithi033.stdout:1119: finishing write tid 2 to smithi03385152-23 2024-09-21T09:54:53.662 INFO:tasks.rados.rados.0.smithi033.stdout:1119: finishing write tid 3 to smithi03385152-23 2024-09-21T09:54:53.662 INFO:tasks.rados.rados.0.smithi033.stdout:1119: finishing write tid 4 to smithi03385152-23 2024-09-21T09:54:53.662 INFO:tasks.rados.rados.0.smithi033.stdout:1119: finishing write tid 5 to smithi03385152-23 2024-09-21T09:54:53.663 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 368 (ObjNum 373 snap 99 seq_num 373) dirty exists 2024-09-21T09:54:53.663 INFO:tasks.rados.rados.0.smithi033.stdout:1119: left oid 23 (ObjNum 373 snap 99 seq_num 373) 2024-09-21T09:54:53.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:53 smithi033 ceph-mon[98546]: pgmap v29: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 632 KiB/s wr, 4 op/s 2024-09-21T09:54:53.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:53 smithi033 ceph-mon[98546]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:54:53.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:53 smithi033 ceph-mon[96568]: pgmap v29: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 632 KiB/s wr, 4 op/s 2024-09-21T09:54:53.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:53 smithi033 ceph-mon[96568]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:54:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:53 smithi105 ceph-mon[31286]: pgmap v29: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 632 KiB/s wr, 4 op/s 2024-09-21T09:54:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:53 smithi105 ceph-mon[31286]: osdmap e268: 8 total, 8 up, 8 in 2024-09-21T09:54:54.408 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:54.434 INFO:tasks.rados.rados.0.smithi033.stdout:1114: done (4 left) 2024-09-21T09:54:54.434 INFO:tasks.rados.rados.0.smithi033.stdout:1117: done (3 left) 2024-09-21T09:54:54.434 INFO:tasks.rados.rados.0.smithi033.stdout:1118: done (2 left) 2024-09-21T09:54:54.434 INFO:tasks.rados.rados.0.smithi033.stdout:1119: done (1 left) 2024-09-21T09:54:54.434 INFO:tasks.rados.rados.0.smithi033.stdout:1120: done (0 left) 2024-09-21T09:54:54.434 INFO:tasks.rados.rados.0.smithi033.stdout:1121: read oid 7 snap 95 2024-09-21T09:54:54.434 INFO:tasks.rados.rados.0.smithi033.stdout:1121: expect (ObjNum 324 snap 84 seq_num 324) 2024-09-21T09:54:54.434 INFO:tasks.rados.rados.0.smithi033.stdout:1122: copy_from oid 16 from oid 4 current snap is 100 2024-09-21T09:54:54.435 INFO:tasks.rados.rados.0.smithi033.stdout:1123: snap_remove snap 97 2024-09-21T09:54:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:54 smithi033 ceph-mon[98546]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:54:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:54 smithi033 ceph-mon[98546]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:54:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:54 smithi033 ceph-mon[98546]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:54 smithi033 ceph-mon[98546]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:54.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:54 smithi033 ceph-mon[96568]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:54:54.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:54 smithi033 ceph-mon[96568]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:54:54.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:54 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:54.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:54 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:54 smithi105 ceph-mon[31286]: osdmap e269: 8 total, 8 up, 8 in 2024-09-21T09:54:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:54 smithi105 ceph-mon[31286]: osdmap e270: 8 total, 8 up, 8 in 2024-09-21T09:54:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:54 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:54 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:55.325 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:54:55.137+0000 7fc38e955640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-21T09:54:55.439 INFO:tasks.rados.rados.0.smithi033.stdout:1123: done (2 left) 2024-09-21T09:54:55.439 INFO:tasks.rados.rados.0.smithi033.stdout:1124: snap_create 2024-09-21T09:54:55.439 INFO:tasks.rados.rados.0.smithi033.stdout:1121: expect (ObjNum 324 snap 84 seq_num 324) 2024-09-21T09:54:55.459 INFO:tasks.rados.rados.0.smithi033.stdout:1122: finishing copy_from racing read to smithi03385152-16 2024-09-21T09:54:55.459 INFO:tasks.rados.rados.0.smithi033.stdout:1122: finishing copy_from to smithi03385152-16 2024-09-21T09:54:55.459 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 157 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:54:55.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:55 smithi105 ceph-mon[31286]: pgmap v33: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2 op/s 2024-09-21T09:54:55.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:55 smithi105 ceph-mon[31286]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:54:55.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:55 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:55.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:55 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:55.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-mon[96568]: pgmap v33: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2 op/s 2024-09-21T09:54:55.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-mon[96568]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:54:55.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:55.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-mon[98546]: pgmap v33: 121 pgs: 121 active+clean; 148 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 2 op/s 2024-09-21T09:54:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-mon[98546]: osdmap e271: 8 total, 8 up, 8 in 2024-09-21T09:54:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-mon[98546]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:55 smithi033 ceph-mon[98546]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:56.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:56.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:56.451 INFO:tasks.rados.rados.0.smithi033.stdout:1121: done (2 left) 2024-09-21T09:54:56.451 INFO:tasks.rados.rados.0.smithi033.stdout:1122: done (1 left) 2024-09-21T09:54:56.452 INFO:tasks.rados.rados.0.smithi033.stdout:1124: done (0 left) 2024-09-21T09:54:56.452 INFO:tasks.rados.rados.0.smithi033.stdout:1125: read oid 11 snap -1 2024-09-21T09:54:56.452 INFO:tasks.rados.rados.0.smithi033.stdout:1125: expect (ObjNum 356 snap 94 seq_num 356) 2024-09-21T09:54:56.452 INFO:tasks.rados.rados.0.smithi033.stdout:1126: copy_from oid 13 from oid 29 current snap is 101 2024-09-21T09:54:56.452 INFO:tasks.rados.rados.0.smithi033.stdout:1127: rollback oid 33 current snap is 101 2024-09-21T09:54:56.452 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 98 2024-09-21T09:54:56.452 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 101 2024-09-21T09:54:56.452 INFO:tasks.rados.rados.0.smithi033.stdout:1128: seq_num 374 ranges {1605632=671744,2277376=778240,3055616=24576} 2024-09-21T09:54:56.465 INFO:tasks.rados.rados.0.smithi033.stdout:1128: writing smithi03385152-2 from 1605632 to 2277376 tid 1 2024-09-21T09:54:56.470 INFO:tasks.rados.rados.0.smithi033.stdout:1128: writing smithi03385152-2 from 2277376 to 3055616 tid 2 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1128: writing smithi03385152-2 from 3055616 to 3080192 tid 3 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1129: delete oid 50 current snap is 101 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1126: finishing copy_from to smithi03385152-13 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1126: got expected ENOENT (src dne) 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1126: finishing copy_from racing read to smithi03385152-13 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1127: finishing rollback tid 0 to smithi03385152-33 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1127: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1127: finishing rollback tid 2 to smithi03385152-33 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 374 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:54:56.471 INFO:tasks.rados.rados.0.smithi033.stdout:1125: expect (ObjNum 356 snap 94 seq_num 356) 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:1128: finishing write tid 1 to smithi03385152-2 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:1128: finishing write tid 2 to smithi03385152-2 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:1128: finishing write tid 3 to smithi03385152-2 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:1128: finishing write tid 4 to smithi03385152-2 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:1128: finishing write tid 5 to smithi03385152-2 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 613 (ObjNum 374 snap 101 seq_num 374) dirty exists 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:1128: left oid 2 (ObjNum 374 snap 101 seq_num 374) 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:1125: done (4 left) 2024-09-21T09:54:56.713 INFO:tasks.rados.rados.0.smithi033.stdout:1126: done (3 left) 2024-09-21T09:54:56.714 INFO:tasks.rados.rados.0.smithi033.stdout:1127: done (2 left) 2024-09-21T09:54:56.714 INFO:tasks.rados.rados.0.smithi033.stdout:1128: done (1 left) 2024-09-21T09:54:56.714 INFO:tasks.rados.rados.0.smithi033.stdout:1129: done (0 left) 2024-09-21T09:54:56.714 INFO:tasks.rados.rados.0.smithi033.stdout:1130: rollback oid 25 current snap is 101 2024-09-21T09:54:56.714 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 25 to 100 2024-09-21T09:54:56.714 INFO:tasks.rados.rados.0.smithi033.stdout:1131: snap_remove snap 96 2024-09-21T09:54:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:56 smithi033 ceph-mon[98546]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:54:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:56 smithi033 ceph-mon[96568]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:54:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:56 smithi105 ceph-mon[31286]: osdmap e272: 8 total, 8 up, 8 in 2024-09-21T09:54:57.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:54:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:54:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:54:57.456 INFO:tasks.rados.rados.0.smithi033.stdout:1131: done (1 left) 2024-09-21T09:54:57.456 INFO:tasks.rados.rados.0.smithi033.stdout:1130: finishing rollback tid 1 to smithi03385152-25 2024-09-21T09:54:57.456 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:54:57.456 INFO:tasks.rados.rados.0.smithi033.stdout:1132: read oid 20 snap -1 2024-09-21T09:54:57.456 INFO:tasks.rados.rados.0.smithi033.stdout:1132: expect deleted 2024-09-21T09:54:57.457 INFO:tasks.rados.rados.0.smithi033.stdout:1130: done (1 left) 2024-09-21T09:54:57.457 INFO:tasks.rados.rados.0.smithi033.stdout:1133: delete oid 1 current snap is 101 2024-09-21T09:54:57.462 INFO:tasks.rados.rados.0.smithi033.stdout:1132: done (1 left) 2024-09-21T09:54:57.462 INFO:tasks.rados.rados.0.smithi033.stdout:1133: done (0 left) 2024-09-21T09:54:57.462 INFO:tasks.rados.rados.0.smithi033.stdout:1134: rollback oid 8 current snap is 101 2024-09-21T09:54:57.463 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 8 to 98 2024-09-21T09:54:57.463 INFO:tasks.rados.rados.0.smithi033.stdout:1135: rmattr oid 2 current snap is 101 2024-09-21T09:54:57.463 INFO:tasks.rados.rados.0.smithi033.stdout:1135: done (1 left) 2024-09-21T09:54:57.463 INFO:tasks.rados.rados.0.smithi033.stdout:1136: delete oid 9 current snap is 101 2024-09-21T09:54:57.466 INFO:tasks.rados.rados.0.smithi033.stdout:1136: done (1 left) 2024-09-21T09:54:57.466 INFO:tasks.rados.rados.0.smithi033.stdout:1137: read oid 16 snap -1 2024-09-21T09:54:57.466 INFO:tasks.rados.rados.0.smithi033.stdout:1137: expect (ObjNum 285 snap 69 seq_num 285) 2024-09-21T09:54:57.466 INFO:tasks.rados.rados.0.smithi033.stdout:1138: snap_create 2024-09-21T09:54:57.467 INFO:tasks.rados.rados.0.smithi033.stdout:1134: finishing rollback tid 0 to smithi03385152-8 2024-09-21T09:54:57.469 INFO:tasks.rados.rados.0.smithi033.stdout:1134: finishing rollback tid 1 to smithi03385152-8 2024-09-21T09:54:57.469 INFO:tasks.rados.rados.0.smithi033.stdout:1134: finishing rollback tid 2 to smithi03385152-8 2024-09-21T09:54:57.469 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 628 (ObjNum 290 snap 70 seq_num 290) dirty exists 2024-09-21T09:54:57.514 INFO:tasks.rados.rados.0.smithi033.stdout:1137: expect (ObjNum 285 snap 69 seq_num 285) 2024-09-21T09:54:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:57 smithi033 ceph-mon[96568]: pgmap v36: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.2 MiB/s wr, 13 op/s 2024-09-21T09:54:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:57 smithi033 ceph-mon[96568]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:54:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:57 smithi033 ceph-mon[98546]: pgmap v36: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.2 MiB/s wr, 13 op/s 2024-09-21T09:54:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:57 smithi033 ceph-mon[98546]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:54:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:57 smithi105 ceph-mon[31286]: pgmap v36: 121 pgs: 121 active+clean; 160 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.6 MiB/s rd, 3.2 MiB/s wr, 13 op/s 2024-09-21T09:54:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:57 smithi105 ceph-mon[31286]: osdmap e273: 8 total, 8 up, 8 in 2024-09-21T09:54:58.467 INFO:tasks.rados.rados.0.smithi033.stdout:1134: done (2 left) 2024-09-21T09:54:58.467 INFO:tasks.rados.rados.0.smithi033.stdout:1137: done (1 left) 2024-09-21T09:54:58.467 INFO:tasks.rados.rados.0.smithi033.stdout:1138: done (0 left) 2024-09-21T09:54:58.467 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 102 2024-09-21T09:54:58.467 INFO:tasks.rados.rados.0.smithi033.stdout:1139: seq_num 378 ranges {2695168=737280,3432448=720896,4153344=81920} 2024-09-21T09:54:58.485 INFO:tasks.rados.rados.0.smithi033.stdout:1139: writing smithi03385152-40 from 2695168 to 3432448 tid 1 2024-09-21T09:54:58.489 INFO:tasks.rados.rados.0.smithi033.stdout:1139: writing smithi03385152-40 from 3432448 to 4153344 tid 2 2024-09-21T09:54:58.490 INFO:tasks.rados.rados.0.smithi033.stdout:1139: writing smithi03385152-40 from 4153344 to 4235264 tid 3 2024-09-21T09:54:58.490 INFO:tasks.rados.rados.0.smithi033.stdout:1140: delete oid 2 current snap is 102 2024-09-21T09:54:58.493 INFO:tasks.rados.rados.0.smithi033.stdout:1140: done (1 left) 2024-09-21T09:54:58.493 INFO:tasks.rados.rados.0.smithi033.stdout:1141: snap_remove snap 100 2024-09-21T09:54:58.640 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:58.641 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:58 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-21T09:54:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:54:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:58.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: osdmap e274: 8 total, 8 up, 8 in 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-21T09:54:58.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:58 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-21T09:54:59.473 INFO:tasks.rados.rados.0.smithi033.stdout:1141: done (1 left) 2024-09-21T09:54:59.473 INFO:tasks.rados.rados.0.smithi033.stdout:1142: copy_from oid 43 from oid 39 current snap is 102 2024-09-21T09:54:59.473 INFO:tasks.rados.rados.0.smithi033.stdout:1139: finishing write tid 1 to smithi03385152-40 2024-09-21T09:54:59.473 INFO:tasks.rados.rados.0.smithi033.stdout:1139: finishing write tid 2 to smithi03385152-40 2024-09-21T09:54:59.473 INFO:tasks.rados.rados.0.smithi033.stdout:1139: finishing write tid 3 to smithi03385152-40 2024-09-21T09:54:59.473 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 102 2024-09-21T09:54:59.473 INFO:tasks.rados.rados.0.smithi033.stdout:1143: seq_num 380 ranges {5505024=770048,6275072=483328,6758400=319488} 2024-09-21T09:54:59.508 INFO:tasks.rados.rados.0.smithi033.stdout:1143: writing smithi03385152-35 from 5505024 to 6275072 tid 1 2024-09-21T09:54:59.510 INFO:tasks.rados.rados.0.smithi033.stdout:1143: writing smithi03385152-35 from 6275072 to 6758400 tid 2 2024-09-21T09:54:59.512 INFO:tasks.rados.rados.0.smithi033.stdout:1143: writing smithi03385152-35 from 6758400 to 7077888 tid 3 2024-09-21T09:54:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:1144: delete oid 11 current snap is 102 2024-09-21T09:54:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:1139: finishing write tid 4 to smithi03385152-40 2024-09-21T09:54:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:1139: finishing write tid 5 to smithi03385152-40 2024-09-21T09:54:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 629 (ObjNum 378 snap 102 seq_num 378) dirty exists 2024-09-21T09:54:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:1139: left oid 40 (ObjNum 378 snap 102 seq_num 378) 2024-09-21T09:54:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:1142: finishing copy_from to smithi03385152-43 2024-09-21T09:54:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:1142: got expected ENOENT (src dne) 2024-09-21T09:54:59.513 INFO:tasks.rados.rados.0.smithi033.stdout:1142: finishing copy_from racing read to smithi03385152-43 2024-09-21T09:54:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:1139: done (3 left) 2024-09-21T09:54:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:1142: done (2 left) 2024-09-21T09:54:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:1144: done (1 left) 2024-09-21T09:54:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:1145: read oid 7 snap 98 2024-09-21T09:54:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:1145: expect (ObjNum 324 snap 84 seq_num 324) 2024-09-21T09:54:59.516 INFO:tasks.rados.rados.0.smithi033.stdout:1146: read oid 6 snap -1 2024-09-21T09:54:59.517 INFO:tasks.rados.rados.0.smithi033.stdout:1146: expect (ObjNum 353 snap 93 seq_num 353) 2024-09-21T09:54:59.517 INFO:tasks.rados.rados.0.smithi033.stdout:1147: read oid 5 snap -1 2024-09-21T09:54:59.517 INFO:tasks.rados.rados.0.smithi033.stdout:1147: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:54:59.517 INFO:tasks.rados.rados.0.smithi033.stdout:1148: setattr oid 28 current snap is 102 2024-09-21T09:54:59.517 INFO:tasks.rados.rados.0.smithi033.stdout:1147: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1143: finishing write tid 1 to smithi03385152-35 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1143: finishing write tid 2 to smithi03385152-35 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1143: finishing write tid 3 to smithi03385152-35 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1143: finishing write tid 4 to smithi03385152-35 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1147: done (4 left) 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 102 2024-09-21T09:54:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1149: seq_num 382 ranges {1253376=802816,2056192=704512,2760704=98304} 2024-09-21T09:54:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:1149: writing smithi03385152-38 from 1253376 to 2056192 tid 1 2024-09-21T09:54:59.539 INFO:tasks.rados.rados.0.smithi033.stdout:1149: writing smithi03385152-38 from 2056192 to 2760704 tid 2 2024-09-21T09:54:59.540 INFO:tasks.rados.rados.0.smithi033.stdout:1149: writing smithi03385152-38 from 2760704 to 2859008 tid 3 2024-09-21T09:54:59.540 INFO:tasks.rados.rados.0.smithi033.stdout:1150: snap_create 2024-09-21T09:54:59.540 INFO:tasks.rados.rados.0.smithi033.stdout:1145: expect (ObjNum 324 snap 84 seq_num 324) 2024-09-21T09:54:59.562 INFO:tasks.rados.rados.0.smithi033.stdout:1143: finishing write tid 5 to smithi03385152-35 2024-09-21T09:54:59.562 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 724 (ObjNum 380 snap 102 seq_num 380) dirty exists 2024-09-21T09:54:59.562 INFO:tasks.rados.rados.0.smithi033.stdout:1143: left oid 35 (ObjNum 380 snap 102 seq_num 380) 2024-09-21T09:54:59.562 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 152 (ObjNum 369 snap 97 seq_num 369) dirty exists 2024-09-21T09:54:59.562 INFO:tasks.rados.rados.0.smithi033.stdout:1149: finishing write tid 1 to smithi03385152-38 2024-09-21T09:54:59.562 INFO:tasks.rados.rados.0.smithi033.stdout:1149: finishing write tid 2 to smithi03385152-38 2024-09-21T09:54:59.562 INFO:tasks.rados.rados.0.smithi033.stdout:1149: finishing write tid 3 to smithi03385152-38 2024-09-21T09:54:59.563 INFO:tasks.rados.rados.0.smithi033.stdout:1149: finishing write tid 4 to smithi03385152-38 2024-09-21T09:54:59.563 INFO:tasks.rados.rados.0.smithi033.stdout:1149: finishing write tid 5 to smithi03385152-38 2024-09-21T09:54:59.563 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 633 (ObjNum 382 snap 102 seq_num 382) dirty exists 2024-09-21T09:54:59.563 INFO:tasks.rados.rados.0.smithi033.stdout:1149: left oid 38 (ObjNum 382 snap 102 seq_num 382) 2024-09-21T09:54:59.563 INFO:tasks.rados.rados.0.smithi033.stdout:1146: expect (ObjNum 353 snap 93 seq_num 353) 2024-09-21T09:54:59.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:59 smithi033 ceph-mon[98546]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:54:59.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:59 smithi033 ceph-mon[98546]: pgmap v39: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 3.5 MiB/s wr, 14 op/s 2024-09-21T09:54:59.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:54:59 smithi033 ceph-mon[98546]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:54:59.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:59 smithi033 ceph-mon[96568]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:54:59.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:59 smithi033 ceph-mon[96568]: pgmap v39: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 3.5 MiB/s wr, 14 op/s 2024-09-21T09:54:59.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:54:59 smithi033 ceph-mon[96568]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:54:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:59 smithi105 ceph-mon[31286]: Upgrade: It appears safe to stop mon.b 2024-09-21T09:54:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:59 smithi105 ceph-mon[31286]: pgmap v39: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 3.5 MiB/s wr, 14 op/s 2024-09-21T09:54:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:54:59 smithi105 ceph-mon[31286]: osdmap e275: 8 total, 8 up, 8 in 2024-09-21T09:55:00.406 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:00.481 INFO:tasks.rados.rados.0.smithi033.stdout:1143: done (5 left) 2024-09-21T09:55:00.481 INFO:tasks.rados.rados.0.smithi033.stdout:1145: done (4 left) 2024-09-21T09:55:00.481 INFO:tasks.rados.rados.0.smithi033.stdout:1146: done (3 left) 2024-09-21T09:55:00.481 INFO:tasks.rados.rados.0.smithi033.stdout:1148: done (2 left) 2024-09-21T09:55:00.481 INFO:tasks.rados.rados.0.smithi033.stdout:1149: done (1 left) 2024-09-21T09:55:00.481 INFO:tasks.rados.rados.0.smithi033.stdout:1150: done (0 left) 2024-09-21T09:55:00.482 INFO:tasks.rados.rados.0.smithi033.stdout:1151: copy_from oid 6 from oid 49 current snap is 103 2024-09-21T09:55:00.482 INFO:tasks.rados.rados.0.smithi033.stdout:1152: rollback oid 33 current snap is 103 2024-09-21T09:55:00.482 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 101 2024-09-21T09:55:00.482 INFO:tasks.rados.rados.0.smithi033.stdout:1153: copy_from oid 38 from oid 1 current snap is 103 2024-09-21T09:55:00.482 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 103 2024-09-21T09:55:00.482 INFO:tasks.rados.rados.0.smithi033.stdout:1154: seq_num 383 ranges {9158656=417792,9576448=589824,10166272=417792,10584064=196608} 2024-09-21T09:55:00.533 INFO:tasks.rados.rados.0.smithi033.stdout:1154: writing smithi03385152-16 from 9158656 to 9576448 tid 1 2024-09-21T09:55:00.537 INFO:tasks.rados.rados.0.smithi033.stdout:1154: writing smithi03385152-16 from 9576448 to 10166272 tid 2 2024-09-21T09:55:00.539 INFO:tasks.rados.rados.0.smithi033.stdout:1154: writing smithi03385152-16 from 10166272 to 10584064 tid 3 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1154: writing smithi03385152-16 from 10584064 to 10780672 tid 4 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1155: snap_create 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1153: finishing copy_from to smithi03385152-38 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1153: got expected ENOENT (src dne) 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1153: finishing copy_from racing read to smithi03385152-38 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1151: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1152: finishing rollback tid 0 to smithi03385152-33 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1152: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1152: finishing rollback tid 2 to smithi03385152-33 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 382 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1151: finishing copy_from to smithi03385152-6 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 433 (ObjNum 347 snap 92 seq_num 347) dirty exists 2024-09-21T09:55:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:1154: finishing write tid 1 to smithi03385152-16 2024-09-21T09:55:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:1154: finishing write tid 2 to smithi03385152-16 2024-09-21T09:55:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:1154: finishing write tid 3 to smithi03385152-16 2024-09-21T09:55:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:1154: finishing write tid 4 to smithi03385152-16 2024-09-21T09:55:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:1154: finishing write tid 5 to smithi03385152-16 2024-09-21T09:55:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:1154: finishing write tid 6 to smithi03385152-16 2024-09-21T09:55:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 730 (ObjNum 383 snap 103 seq_num 383) dirty exists 2024-09-21T09:55:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:1154: left oid 16 (ObjNum 383 snap 103 seq_num 383) 2024-09-21T09:55:00.739 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:00 smithi105 ceph-mon[31286]: Upgrade: Updating mon.b 2024-09-21T09:55:00.739 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:00 smithi105 ceph-mon[31286]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:55:00.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:00 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:55:00.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:00 smithi105 ceph-mon[31286]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:55:00.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:00 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:55:00.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:00 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:00.740 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:00 smithi105 ceph-mon[31286]: Deploying daemon mon.b on smithi105 2024-09-21T09:55:00.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[98546]: Upgrade: Updating mon.b 2024-09-21T09:55:00.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[98546]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:55:00.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:55:00.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[98546]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:55:00.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:55:00.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:00.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[98546]: Deploying daemon mon.b on smithi105 2024-09-21T09:55:00.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[96568]: Upgrade: Updating mon.b 2024-09-21T09:55:00.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[96568]: from='mgr.25157 ' entity='mgr.y' 2024-09-21T09:55:00.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-21T09:55:00.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[96568]: osdmap e276: 8 total, 8 up, 8 in 2024-09-21T09:55:00.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-21T09:55:00.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-21T09:55:00.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:00 smithi033 ceph-mon[96568]: Deploying daemon mon.b on smithi105 2024-09-21T09:55:01.494 INFO:tasks.rados.rados.0.smithi033.stdout:1151: done (4 left) 2024-09-21T09:55:01.494 INFO:tasks.rados.rados.0.smithi033.stdout:1152: done (3 left) 2024-09-21T09:55:01.494 INFO:tasks.rados.rados.0.smithi033.stdout:1153: done (2 left) 2024-09-21T09:55:01.494 INFO:tasks.rados.rados.0.smithi033.stdout:1154: done (1 left) 2024-09-21T09:55:01.494 INFO:tasks.rados.rados.0.smithi033.stdout:1155: done (0 left) 2024-09-21T09:55:01.495 INFO:tasks.rados.rados.0.smithi033.stdout:1156: copy_from oid 31 from oid 47 current snap is 104 2024-09-21T09:55:01.495 INFO:tasks.rados.rados.0.smithi033.stdout:1157: read oid 12 snap 101 2024-09-21T09:55:01.495 INFO:tasks.rados.rados.0.smithi033.stdout:1157: expect (ObjNum 365 snap 96 seq_num 365) 2024-09-21T09:55:01.495 INFO:tasks.rados.rados.0.smithi033.stdout:1158: delete oid 32 current snap is 104 2024-09-21T09:55:01.498 INFO:tasks.rados.rados.0.smithi033.stdout:1156: finishing copy_from to smithi03385152-31 2024-09-21T09:55:01.498 INFO:tasks.rados.rados.0.smithi033.stdout:1156: got expected ENOENT (src dne) 2024-09-21T09:55:01.499 INFO:tasks.rados.rados.0.smithi033.stdout:1156: finishing copy_from racing read to smithi03385152-31 2024-09-21T09:55:01.499 INFO:tasks.rados.rados.0.smithi033.stdout:1156: done (2 left) 2024-09-21T09:55:01.499 INFO:tasks.rados.rados.0.smithi033.stdout:1158: done (1 left) 2024-09-21T09:55:01.499 INFO:tasks.rados.rados.0.smithi033.stdout:1159: snap_create 2024-09-21T09:55:01.504 INFO:tasks.rados.rados.0.smithi033.stdout:1157: expect (ObjNum 365 snap 96 seq_num 365) 2024-09-21T09:55:01.613 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:01 smithi105 systemd[1]: Stopping Ceph mon.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:55:01.614 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:01 smithi105 ceph-mon[31286]: pgmap v42: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 356 KiB/s wr, 1 op/s 2024-09-21T09:55:01.614 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:01 smithi105 ceph-mon[31286]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:55:01.614 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:01 smithi105 ceph-mon[31286]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:01.634 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:01 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b[31262]: 2024-09-21T09:55:01.610+0000 7f730526d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:55:01.634 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:01 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b[31262]: 2024-09-21T09:55:01.610+0000 7f730526d640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-21T09:55:01.634 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:01 smithi105 podman[86458]: 2024-09-21 09:55:01.631566821 +0000 UTC m=+0.132051803 container died c2741af3f853eba26e6ce9d6bec37c7f672802e9f6e40cb60e5f7a0f3cad5a74 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T09:55:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:01 smithi033 ceph-mon[96568]: pgmap v42: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 356 KiB/s wr, 1 op/s 2024-09-21T09:55:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:01 smithi033 ceph-mon[96568]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:55:01.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:01.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:55:01] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-09-21T09:55:01.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:55:01] ENGINE Bus STOPPING 2024-09-21T09:55:01.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:01 smithi033 ceph-mon[98546]: pgmap v42: 121 pgs: 121 active+clean; 162 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 356 KiB/s wr, 1 op/s 2024-09-21T09:55:01.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:01 smithi033 ceph-mon[98546]: osdmap e277: 8 total, 8 up, 8 in 2024-09-21T09:55:01.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-21T09:55:02.270 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 podman[86458]: 2024-09-21 09:55:02.025337094 +0000 UTC m=+0.525822084 container cleanup c2741af3f853eba26e6ce9d6bec37c7f672802e9f6e40cb60e5f7a0f3cad5a74 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-21T09:55:02.270 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 bash[86458]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b 2024-09-21T09:55:02.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:02 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:55:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T09:55:02.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:02 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:55:02] ENGINE Bus STOPPED 2024-09-21T09:55:02.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:02 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:55:02] ENGINE Bus STARTING 2024-09-21T09:55:02.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:02 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:55:02] ENGINE Serving on http://:::9283 2024-09-21T09:55:02.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:02 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:09:55:02] ENGINE Bus STARTED 2024-09-21T09:55:02.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 podman[86476]: 2024-09-21 09:55:02.268283956 +0000 UTC m=+0.636185340 container remove c2741af3f853eba26e6ce9d6bec37c7f672802e9f6e40cb60e5f7a0f3cad5a74 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T09:55:02.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.b.service: Deactivated successfully. 2024-09-21T09:55:02.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 systemd[1]: Stopped Ceph mon.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:55:02.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.b.service: Consumed 9.334s CPU time. 2024-09-21T09:55:03.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 systemd[1]: Starting Ceph mon.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:55:03.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 podman[86611]: 2024-09-21 09:55:02.770831369 +0000 UTC m=+0.015776960 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:55:03.020 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:02 smithi105 podman[86611]: 2024-09-21 09:55:02.877530191 +0000 UTC m=+0.122475814 container create 9f90671a2e298396ed70122867b7702c03ce28b5680d5296540545d8285e6740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-21T09:55:03.436 INFO:tasks.rados.rados.0.smithi033.stdout:1157: done (1 left) 2024-09-21T09:55:03.436 INFO:tasks.rados.rados.0.smithi033.stdout:1159: done (0 left) 2024-09-21T09:55:03.436 INFO:tasks.rados.rados.0.smithi033.stdout:1160: copy_from oid 38 from oid 8 current snap is 105 2024-09-21T09:55:03.436 INFO:tasks.rados.rados.0.smithi033.stdout:1161: delete oid 42 current snap is 105 2024-09-21T09:55:03.444 INFO:tasks.rados.rados.0.smithi033.stdout:1161: done (1 left) 2024-09-21T09:55:03.444 INFO:tasks.rados.rados.0.smithi033.stdout:1162: delete oid 29 current snap is 105 2024-09-21T09:55:03.450 INFO:tasks.rados.rados.0.smithi033.stdout:1162: done (1 left) 2024-09-21T09:55:03.450 INFO:tasks.rados.rados.0.smithi033.stdout:1163: rollback oid 27 current snap is 105 2024-09-21T09:55:03.450 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 27 to 95 2024-09-21T09:55:03.450 INFO:tasks.rados.rados.0.smithi033.stdout:1164: snap_remove snap 104 2024-09-21T09:55:03.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 podman[86611]: 2024-09-21 09:55:03.130000781 +0000 UTC m=+0.374946358 container init 9f90671a2e298396ed70122867b7702c03ce28b5680d5296540545d8285e6740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b) 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 podman[86611]: 2024-09-21 09:55:03.134795547 +0000 UTC m=+0.379741124 container start 9f90671a2e298396ed70122867b7702c03ce28b5680d5296540545d8285e6740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: set uid:gid to 167:167 (ceph:ceph) 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev), process ceph-mon, pid 7 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: pidfile_write: ignore empty --pid-file 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: load: jerasure load: lrc 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: RocksDB version: 7.9.2 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Git sha 0 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Compile date 2024-09-20 20:32:28 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: DB SUMMARY 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: DB Session ID: ZKGMNQY7Q1X1DHH7RFKC 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: CURRENT file: CURRENT 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: IDENTITY file: IDENTITY 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1096 Bytes 2024-09-21T09:55:03.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 12313789 ; 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.error_if_exists: 0 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.create_if_missing: 0 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.paranoid_checks: 1 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.env: 0x562b55333120 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.fs: PosixFileSystem 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.info_log: 0x562b57951700 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_file_opening_threads: 16 2024-09-21T09:55:03.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.statistics: (nil) 2024-09-21T09:55:03.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.use_fsync: 0 2024-09-21T09:55:03.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_log_file_size: 0 2024-09-21T09:55:03.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-21T09:55:03.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-21T09:55:03.533 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.keep_log_file_num: 1000 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.recycle_log_file_num: 0 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.allow_fallocate: 1 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.allow_mmap_reads: 0 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.allow_mmap_writes: 0 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.use_direct_reads: 0 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.create_missing_column_families: 0 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.db_log_dir: 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.wal_dir: 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-21T09:55:03.534 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.advise_random_on_open: 1 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.db_write_buffer_size: 0 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.write_buffer_manager: 0x562b579541e0 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.rate_limiter: (nil) 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.wal_recovery_mode: 2 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.enable_thread_tracking: 0 2024-09-21T09:55:03.535 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.enable_pipelined_write: 0 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.unordered_write: 0 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.row_cache: None 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.wal_filter: None 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.allow_ingest_behind: 0 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.two_write_queues: 0 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.manual_wal_flush: 0 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.wal_compression: 0 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.atomic_flush: 0 2024-09-21T09:55:03.536 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.log_readahead_size: 0 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.best_efforts_recovery: 0 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.allow_data_in_errors: 0 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.db_host_id: __hostname__ 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_background_jobs: 2 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_background_compactions: -1 2024-09-21T09:55:03.537 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_subcompactions: 1 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_total_wal_size: 0 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_open_files: -1 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bytes_per_sync: 0 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_readahead_size: 0 2024-09-21T09:55:03.538 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_background_flushes: -1 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Compression algorithms supported: 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: kZSTD supported: 0 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: kXpressCompression supported: 0 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: kBZip2Compression supported: 0 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: kLZ4Compression supported: 1 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: kZlibCompression supported: 1 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: kLZ4HCCompression supported: 1 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: kSnappyCompression supported: 1 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-21T09:55:03.539 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.merge_operator: 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_filter: None 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_filter_factory: None 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.sst_partitioner_factory: None 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562b57951320) 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout: cache_index_and_filter_blocks: 1 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout: pin_top_level_index_and_filter: 1 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_type: 0 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_block_index_type: 0 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_shortening: 1 2024-09-21T09:55:03.540 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: checksum: 4 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: no_block_cache: 0 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache: 0x562b57945610 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_name: BinnedLRUCache 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_options: 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: capacity : 536870912 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: num_shard_bits : 4 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: strict_capacity_limit : 0 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: high_pri_pool_ratio: 0.000 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_cache_compressed: (nil) 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: persistent_cache: (nil) 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_size: 4096 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_size_deviation: 10 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_restart_interval: 16 2024-09-21T09:55:03.541 INFO:journalctl@ceph.mon.b.smithi105.stdout: index_block_restart_interval: 1 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: metadata_block_size: 4096 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: partition_filters: 0 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: use_delta_encoding: 1 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: filter_policy: bloomfilter 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: whole_key_filtering: 1 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: verify_compression: 0 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: read_amp_bytes_per_bit: 0 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: format_version: 5 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_index_compression: 1 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: block_align: 0 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: max_auto_readahead_size: 262144 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: prepopulate_block_cache: 0 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: initial_auto_readahead_size: 8192 2024-09-21T09:55:03.542 INFO:journalctl@ceph.mon.b.smithi105.stdout: num_file_reads_for_auto_readahead: 2 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.write_buffer_size: 33554432 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_write_buffer_number: 2 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression: NoCompression 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression: Disabled 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.prefix_extractor: nullptr 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.num_levels: 7 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-21T09:55:03.543 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.level: 32767 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.strategy: 0 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.enabled: false 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-21T09:55:03.544 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.target_file_size_base: 67108864 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-21T09:55:03.545 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-21T09:55:03.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-21T09:55:03.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-21T09:55:03.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-21T09:55:03.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-21T09:55:03.546 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.arena_block_size: 1048576 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.disable_auto_compactions: 0 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-21T09:55:03.547 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.inplace_update_support: 0 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.bloom_locality: 0 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.max_successive_merges: 0 2024-09-21T09:55:03.548 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.paranoid_file_checks: 0 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.force_consistency_checks: 1 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.report_bg_io_stats: 0 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.ttl: 2592000 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.enable_blob_files: false 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.min_blob_size: 0 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.blob_file_size: 268435456 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-21T09:55:03.549 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.blob_file_starting_level: 0 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13494, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 16b8ce14-427b-4ab2-af67-5cd8a8d8b474 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912503160508, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912503187336, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 7623187, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13491, "largest_seqno": 17553, "table_properties": {"data_size": 7605088, "index_size": 12636, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4421, "raw_key_size": 40268, "raw_average_key_size": 22, "raw_value_size": 7568000, "raw_average_value_size": 4312, "num_data_blocks": 547, "num_entries": 1755, "num_filter_entries": 1755, "num_deletions": 314, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726912503, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "16b8ce14-427b-4ab2-af67-5cd8a8d8b474", "db_session_id": "ZKGMNQY7Q1X1DHH7RFKC", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726912503187517, "job": 1, "event": "recovery_finished"} 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-21T09:55:03.550 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562b57974e00 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: DB pointer 0x562b57a76000 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** DB Stats ** 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-21T09:55:03.551 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: L0 1/0 7.27 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 663.1 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: L6 1/0 7.64 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: Sum 2/0 14.91 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 663.1 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 663.1 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** Compaction Stats [default] ** 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 663.1 0.01 0.00 1 0.011 0 0 0.0 0.0 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.552 INFO:journalctl@ceph.mon.b.smithi105.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: Flush(GB): cumulative 0.007, interval 0.007 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: Cumulative compaction: 0.01 GB write, 232.08 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: Interval compaction: 0.01 GB write, 232.08 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: Block cache BinnedLRUCache@0x562b57945610#7 capacity: 512.00 MB usage: 17.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,4.53 KB,0.000864267%) IndexBlock(1,13.08 KB,0.00249445%) Misc(1,0.00 KB,0%) 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-21T09:55:03.553 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: starting mon.b rank 1 at public addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] at bind addrs [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: mon.b@-1(???) e3 preinit fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: mon.b@-1(???).mds e7 new map 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: mon.b@-1(???).mds e7 print_map 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: e7 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: legacy client fscid: 1 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: Filesystem 'cephfs' (1) 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: fs_name cephfs 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: epoch 7 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-21T09:55:03.554 INFO:journalctl@ceph.mon.b.smithi105.stdout: created 2024-09-21T09:47:25.959476+0000 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: modified 2024-09-21T09:47:39.447745+0000 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: tableserver 0 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: root 0 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: session_timeout 60 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: session_autoclose 300 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: max_file_size 1099511627776 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: max_xattr_size 65536 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: required_client_features {} 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: last_failure 0 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: last_failure_osd_epoch 0 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: max_mds 1 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: in 0 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: up {0=24358} 2024-09-21T09:55:03.555 INFO:journalctl@ceph.mon.b.smithi105.stdout: failed 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: damaged 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: stopped 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: data_pools [3] 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: metadata_pool 2 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: inline_data disabled 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: balancer 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: bal_rank_mask -1 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: standby_count_wanted 0 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: qdb_cluster leader: 0 members: 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: [mds.b{0:24358} state up:active seq 5 addr [v2:172.21.15.105:6832/2691139879,v1:172.21.15.105:6833/2691139879] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: Standby daemons: 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: 2024-09-21T09:55:03.556 INFO:journalctl@ceph.mon.b.smithi105.stdout: [mds.a{-1:24323} state up:standby seq 1 addr [v2:172.21.15.33:6834/125068843,v1:172.21.15.33:6835/125068843] compat {c=[1],r=[1],i=[7ff]}] 2024-09-21T09:55:03.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: mon.b@-1(???).osd e278 crush map has features 3314933069573799936, adjusting msgr requires 2024-09-21T09:55:03.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: mon.b@-1(???).osd e278 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:55:03.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: mon.b@-1(???).osd e278 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:55:03.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: mon.b@-1(???).osd e278 crush map has features 432629308056666112, adjusting msgr requires 2024-09-21T09:55:03.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 ceph-mon[86649]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-09-21T09:55:03.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 bash[86611]: 9f90671a2e298396ed70122867b7702c03ce28b5680d5296540545d8285e6740 2024-09-21T09:55:03.557 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:03 smithi105 systemd[1]: Started Ceph mon.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:55:04.436 INFO:tasks.rados.rados.0.smithi033.stdout:1164: done (2 left) 2024-09-21T09:55:04.437 INFO:tasks.rados.rados.0.smithi033.stdout:1165: snap_remove snap 102 2024-09-21T09:55:04.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:04.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: overall HEALTH_OK 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: mon.a calling monitor election 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: mon.b calling monitor election 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: mon.c calling monitor election 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: monmap epoch 4 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: last_changed 2024-09-21T09:55:03.248619+0000 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:55:04.531 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: min_mon_release 19 (squid) 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: election_strategy: 1 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: overall HEALTH_OK 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:04.532 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:04 smithi105 ceph-mon[86649]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: overall HEALTH_OK 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: mon.a calling monitor election 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: mon.b calling monitor election 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: mon.c calling monitor election 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: monmap epoch 4 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:55:04.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: last_changed 2024-09-21T09:55:03.248619+0000 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: min_mon_release 19 (squid) 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: election_strategy: 1 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: overall HEALTH_OK 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:04.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:04.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:04.749 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[98546]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:55:04.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:55:04.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:55:04.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-21T09:55:04.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: overall HEALTH_OK 2024-09-21T09:55:04.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: mon.a calling monitor election 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: mon.b calling monitor election 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: mon.c calling monitor election 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: monmap epoch 4 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: fsid a29e8460-77fd-11ef-baf6-efdc52797490 2024-09-21T09:55:04.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: last_changed 2024-09-21T09:55:03.248619+0000 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: created 2024-09-21T09:41:48.442447+0000 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: min_mon_release 19 (squid) 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: election_strategy: 1 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: 0: [v2:172.21.15.33:3300/0,v1:172.21.15.33:6789/0] mon.a 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: 1: [v2:172.21.15.105:3300/0,v1:172.21.15.105:6789/0] mon.b 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: 2: [v2:172.21.15.33:3301/0,v1:172.21.15.33:6790/0] mon.c 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: osdmap e278: 8 total, 8 up, 8 in 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: mgrmap e43: y(active, since 32s), standbys: x 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: overall HEALTH_OK 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:04.751 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:04 smithi033 ceph-mon[96568]: osdmap e279: 8 total, 8 up, 8 in 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1160: finishing copy_from racing read to smithi03385152-38 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1163: finishing rollback tid 0 to smithi03385152-27 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1160: finishing copy_from to smithi03385152-38 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 628 (ObjNum 290 snap 70 seq_num 290) dirty exists 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1163: finishing rollback tid 1 to smithi03385152-27 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1163: finishing rollback tid 2 to smithi03385152-27 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 436 (ObjNum 349 snap 92 seq_num 349) dirty exists 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1160: done (2 left) 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1163: done (1 left) 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1165: done (0 left) 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 105 2024-09-21T09:55:05.443 INFO:tasks.rados.rados.0.smithi033.stdout:1166: seq_num 387 ranges {4898816=483328,5382144=524288,5906432=679936} 2024-09-21T09:55:05.473 INFO:tasks.rados.rados.0.smithi033.stdout:1166: writing smithi03385152-30 from 4898816 to 5382144 tid 1 2024-09-21T09:55:05.476 INFO:tasks.rados.rados.0.smithi033.stdout:1166: writing smithi03385152-30 from 5382144 to 5906432 tid 2 2024-09-21T09:55:05.480 INFO:tasks.rados.rados.0.smithi033.stdout:1166: writing smithi03385152-30 from 5906432 to 6586368 tid 3 2024-09-21T09:55:05.480 INFO:tasks.rados.rados.0.smithi033.stdout:1167: rollback oid 44 current snap is 105 2024-09-21T09:55:05.480 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 44 to 98 2024-09-21T09:55:05.480 INFO:tasks.rados.rados.0.smithi033.stdout:1166: finishing write tid 1 to smithi03385152-30 2024-09-21T09:55:05.480 INFO:tasks.rados.rados.0.smithi033.stdout:1166: finishing write tid 2 to smithi03385152-30 2024-09-21T09:55:05.480 INFO:tasks.rados.rados.0.smithi033.stdout:1168: snap_remove snap 95 2024-09-21T09:55:05.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:05 smithi105 ceph-mon[86649]: pgmap v47: 121 pgs: 121 active+clean; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T09:55:05.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:05 smithi105 ceph-mon[86649]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:55:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:05 smithi033 ceph-mon[98546]: pgmap v47: 121 pgs: 121 active+clean; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T09:55:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:05 smithi033 ceph-mon[98546]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:55:05.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:05 smithi033 ceph-mon[96568]: pgmap v47: 121 pgs: 121 active+clean; 170 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T09:55:05.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:05 smithi033 ceph-mon[96568]: osdmap e280: 8 total, 8 up, 8 in 2024-09-21T09:55:06.024 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:06.024 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:1168: done (2 left) 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:1169: copy_from oid 4 from oid 8 current snap is 105 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:1166: finishing write tid 3 to smithi03385152-30 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:1167: finishing rollback tid 1 to smithi03385152-44 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 0 (ObjNum 97 snap 0 seq_num 0) dirty dne 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:1166: finishing write tid 4 to smithi03385152-30 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:1166: finishing write tid 5 to smithi03385152-30 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 738 (ObjNum 387 snap 105 seq_num 387) dirty exists 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:1166: left oid 30 (ObjNum 387 snap 105 seq_num 387) 2024-09-21T09:55:06.458 INFO:tasks.rados.rados.0.smithi033.stdout:1166: done (2 left) 2024-09-21T09:55:06.459 INFO:tasks.rados.rados.0.smithi033.stdout:1167: done (1 left) 2024-09-21T09:55:06.459 INFO:tasks.rados.rados.0.smithi033.stdout:1170: snap_create 2024-09-21T09:55:06.476 INFO:tasks.rados.rados.0.smithi033.stdout:1169: finishing copy_from racing read to smithi03385152-4 2024-09-21T09:55:06.477 INFO:tasks.rados.rados.0.smithi033.stdout:1169: finishing copy_from to smithi03385152-4 2024-09-21T09:55:06.477 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 628 (ObjNum 290 snap 70 seq_num 290) dirty exists 2024-09-21T09:55:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:06 smithi033 ceph-mon[98546]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:55:06.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:06 smithi033 ceph-mon[96568]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:55:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:06 smithi105 ceph-mon[86649]: osdmap e281: 8 total, 8 up, 8 in 2024-09-21T09:55:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:07.459 INFO:tasks.rados.rados.0.smithi033.stdout:1169: done (1 left) 2024-09-21T09:55:07.459 INFO:tasks.rados.rados.0.smithi033.stdout:1170: done (0 left) 2024-09-21T09:55:07.459 INFO:tasks.rados.rados.0.smithi033.stdout:1171: copy_from oid 35 from oid 22 current snap is 106 2024-09-21T09:55:07.459 INFO:tasks.rados.rados.0.smithi033.stdout:1172: rollback oid 20 current snap is 106 2024-09-21T09:55:07.459 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 105 2024-09-21T09:55:07.459 INFO:tasks.rados.rados.0.smithi033.stdout:1173: delete oid 27 current snap is 106 2024-09-21T09:55:07.463 INFO:tasks.rados.rados.0.smithi033.stdout:1173: done (2 left) 2024-09-21T09:55:07.464 INFO:tasks.rados.rados.0.smithi033.stdout:1174: rollback oid 36 current snap is 106 2024-09-21T09:55:07.464 INFO:tasks.rados.rados.0.smithi033.stdout:1172: finishing rollback tid 1 to smithi03385152-20 2024-09-21T09:55:07.464 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 0 (ObjNum 321 snap 0 seq_num 91212992) dirty dne 2024-09-21T09:55:07.464 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 99 2024-09-21T09:55:07.464 INFO:tasks.rados.rados.0.smithi033.stdout:1172: done (2 left) 2024-09-21T09:55:07.464 INFO:tasks.rados.rados.0.smithi033.stdout:1175: delete oid 11 current snap is 106 2024-09-21T09:55:07.468 INFO:tasks.rados.rados.0.smithi033.stdout:1175: done (2 left) 2024-09-21T09:55:07.468 INFO:tasks.rados.rados.0.smithi033.stdout:1176: copy_from oid 20 from oid 34 current snap is 106 2024-09-21T09:55:07.468 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 106 2024-09-21T09:55:07.468 INFO:tasks.rados.rados.0.smithi033.stdout:1177: seq_num 390 ranges {630784=540672,1171456=630784,1802240=573440} 2024-09-21T09:55:07.474 INFO:tasks.rados.rados.0.smithi033.stdout:1177: writing smithi03385152-7 from 630784 to 1171456 tid 1 2024-09-21T09:55:07.478 INFO:tasks.rados.rados.0.smithi033.stdout:1177: writing smithi03385152-7 from 1171456 to 1802240 tid 2 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:1177: writing smithi03385152-7 from 1802240 to 2375680 tid 3 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 106 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:1174: finishing rollback tid 0 to smithi03385152-36 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:1174: finishing rollback tid 1 to smithi03385152-36 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:1174: finishing rollback tid 2 to smithi03385152-36 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 752 (ObjNum 357 snap 94 seq_num 357) dirty exists 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:1176: finishing copy_from racing read to smithi03385152-20 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:1176: finishing copy_from to smithi03385152-20 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 676 (ObjNum 313 snap 83 seq_num 313) dirty exists 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:1171: finishing copy_from racing read to smithi03385152-35 2024-09-21T09:55:07.483 INFO:tasks.rados.rados.0.smithi033.stdout:1178: seq_num 391 ranges {0=557056,557056=516096,1073152=688128} 2024-09-21T09:55:07.486 INFO:tasks.rados.rados.0.smithi033.stdout:1178: writing smithi03385152-3 from 0 to 557056 tid 1 2024-09-21T09:55:07.489 INFO:tasks.rados.rados.0.smithi033.stdout:1178: writing smithi03385152-3 from 557056 to 1073152 tid 2 2024-09-21T09:55:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:1178: writing smithi03385152-3 from 1073152 to 1761280 tid 3 2024-09-21T09:55:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:1174: done (4 left) 2024-09-21T09:55:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:1176: done (3 left) 2024-09-21T09:55:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:1179: rollback oid 33 current snap is 106 2024-09-21T09:55:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 101 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:1178: finishing write tid 1 to smithi03385152-3 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:1171: finishing copy_from to smithi03385152-35 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 642 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:1177: finishing write tid 1 to smithi03385152-7 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:1177: finishing write tid 2 to smithi03385152-7 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:1171: done (3 left) 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:1180: read oid 10 snap -1 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:1180: expect (ObjNum 250 snap 65 seq_num 250) 2024-09-21T09:55:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:1181: snap_remove snap 99 2024-09-21T09:55:07.568 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:07 smithi105 ceph-mon[86649]: pgmap v50: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 802 KiB/s rd, 864 KiB/s wr, 3 op/s 2024-09-21T09:55:07.568 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:07 smithi105 ceph-mon[86649]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:55:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:07 smithi033 ceph-mon[98546]: pgmap v50: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 802 KiB/s rd, 864 KiB/s wr, 3 op/s 2024-09-21T09:55:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:07 smithi033 ceph-mon[98546]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:55:07.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:07 smithi033 ceph-mon[96568]: pgmap v50: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 802 KiB/s rd, 864 KiB/s wr, 3 op/s 2024-09-21T09:55:07.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:07 smithi033 ceph-mon[96568]: osdmap e282: 8 total, 8 up, 8 in 2024-09-21T09:55:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1177: finishing write tid 3 to smithi03385152-7 2024-09-21T09:55:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1178: finishing write tid 2 to smithi03385152-3 2024-09-21T09:55:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1177: finishing write tid 4 to smithi03385152-7 2024-09-21T09:55:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1181: done (4 left) 2024-09-21T09:55:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1182: rmattr oid 21 current snap is 106 2024-09-21T09:55:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1177: finishing write tid 5 to smithi03385152-7 2024-09-21T09:55:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 759 (ObjNum 390 snap 106 seq_num 390) dirty exists 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1177: left oid 7 (ObjNum 390 snap 106 seq_num 390) 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1178: finishing write tid 3 to smithi03385152-3 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1178: finishing write tid 4 to smithi03385152-3 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1179: finishing rollback tid 0 to smithi03385152-33 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1179: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1179: finishing rollback tid 2 to smithi03385152-33 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 440 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1178: finishing write tid 5 to smithi03385152-3 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 688 (ObjNum 391 snap 106 seq_num 391) dirty exists 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1178: left oid 3 (ObjNum 391 snap 106 seq_num 391) 2024-09-21T09:55:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1180: expect (ObjNum 250 snap 65 seq_num 250) 2024-09-21T09:55:08.470 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:55:08.200+0000 7fc38e955640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-21T09:55:08.526 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 677 (ObjNum 308 snap 81 seq_num 308) dirty exists 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1177: done (4 left) 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1178: done (3 left) 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1179: done (2 left) 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1180: done (1 left) 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1182: done (0 left) 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1183: read oid 13 snap 98 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1183: expect deleted 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 106 2024-09-21T09:55:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1184: seq_num 392 ranges {450560=573440,1024000=802816,1826816=401408} 2024-09-21T09:55:08.532 INFO:tasks.rados.rados.0.smithi033.stdout:1184: writing smithi03385152-20 from 450560 to 1024000 tid 1 2024-09-21T09:55:08.536 INFO:tasks.rados.rados.0.smithi033.stdout:1184: writing smithi03385152-20 from 1024000 to 1826816 tid 2 2024-09-21T09:55:08.539 INFO:tasks.rados.rados.0.smithi033.stdout:1184: writing smithi03385152-20 from 1826816 to 2228224 tid 3 2024-09-21T09:55:08.539 INFO:tasks.rados.rados.0.smithi033.stdout:1185: snap_create 2024-09-21T09:55:08.539 INFO:tasks.rados.rados.0.smithi033.stdout:1184: finishing write tid 1 to smithi03385152-20 2024-09-21T09:55:08.541 INFO:tasks.rados.rados.0.smithi033.stdout:1184: finishing write tid 2 to smithi03385152-20 2024-09-21T09:55:08.541 INFO:tasks.rados.rados.0.smithi033.stdout:1184: finishing write tid 3 to smithi03385152-20 2024-09-21T09:55:08.541 INFO:tasks.rados.rados.0.smithi033.stdout:1184: finishing write tid 4 to smithi03385152-20 2024-09-21T09:55:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1184: finishing write tid 5 to smithi03385152-20 2024-09-21T09:55:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 681 (ObjNum 392 snap 106 seq_num 392) dirty exists 2024-09-21T09:55:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1184: left oid 20 (ObjNum 392 snap 106 seq_num 392) 2024-09-21T09:55:08.736 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:08 smithi105 ceph-mon[86649]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:55:08.736 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.736 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.736 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.737 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[98546]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[96568]: osdmap e283: 8 total, 8 up, 8 in 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:08.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:09.476 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:55:09.482 INFO:tasks.rados.rados.0.smithi033.stdout:1183: done (2 left) 2024-09-21T09:55:09.482 INFO:tasks.rados.rados.0.smithi033.stdout:1184: done (1 left) 2024-09-21T09:55:09.482 INFO:tasks.rados.rados.0.smithi033.stdout:1185: done (0 left) 2024-09-21T09:55:09.482 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 107 2024-09-21T09:55:09.482 INFO:tasks.rados.rados.0.smithi033.stdout:1186: seq_num 393 ranges {0=589824,589824=679936,1269760=524288} 2024-09-21T09:55:09.486 INFO:tasks.rados.rados.0.smithi033.stdout:1186: writing smithi03385152-42 from 0 to 589824 tid 1 2024-09-21T09:55:09.492 INFO:tasks.rados.rados.0.smithi033.stdout:1186: writing smithi03385152-42 from 589824 to 1269760 tid 2 2024-09-21T09:55:09.496 INFO:tasks.rados.rados.0.smithi033.stdout:1186: writing smithi03385152-42 from 1269760 to 1794048 tid 3 2024-09-21T09:55:09.496 INFO:tasks.rados.rados.0.smithi033.stdout:1187: read oid 6 snap -1 2024-09-21T09:55:09.496 INFO:tasks.rados.rados.0.smithi033.stdout:1187: expect (ObjNum 347 snap 92 seq_num 347) 2024-09-21T09:55:09.496 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 107 2024-09-21T09:55:09.496 INFO:tasks.rados.rados.0.smithi033.stdout:1186: finishing write tid 1 to smithi03385152-42 2024-09-21T09:55:09.497 INFO:tasks.rados.rados.0.smithi033.stdout:1186: finishing write tid 2 to smithi03385152-42 2024-09-21T09:55:09.497 INFO:tasks.rados.rados.0.smithi033.stdout:1188: seq_num 394 ranges {10543104=606208,11149312=565248,11714560=557056,12271616=81920} 2024-09-21T09:55:09.559 INFO:tasks.rados.rados.0.smithi033.stdout:1188: writing smithi03385152-14 from 10543104 to 11149312 tid 1 2024-09-21T09:55:09.562 INFO:tasks.rados.rados.0.smithi033.stdout:1188: writing smithi03385152-14 from 11149312 to 11714560 tid 2 2024-09-21T09:55:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:1188: writing smithi03385152-14 from 11714560 to 12271616 tid 3 2024-09-21T09:55:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:1188: writing smithi03385152-14 from 12271616 to 12353536 tid 4 2024-09-21T09:55:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 107 2024-09-21T09:55:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:1189: seq_num 395 ranges {1318912=622592,1941504=450560,2392064=753664} 2024-09-21T09:55:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:1189: writing smithi03385152-12 from 1318912 to 1941504 tid 1 2024-09-21T09:55:09.579 INFO:tasks.rados.rados.0.smithi033.stdout:1189: writing smithi03385152-12 from 1941504 to 2392064 tid 2 2024-09-21T09:55:09.584 INFO:tasks.rados.rados.0.smithi033.stdout:1189: writing smithi03385152-12 from 2392064 to 3145728 tid 3 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:1190: rmattr oid 16 current snap is 107 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:1190: done (4 left) 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:1191: delete oid 29 current snap is 107 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:1186: finishing write tid 3 to smithi03385152-42 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:1186: finishing write tid 4 to smithi03385152-42 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:1186: finishing write tid 5 to smithi03385152-42 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 638 (ObjNum 393 snap 107 seq_num 393) dirty exists 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:1186: left oid 42 (ObjNum 393 snap 107 seq_num 393) 2024-09-21T09:55:09.585 INFO:tasks.rados.rados.0.smithi033.stdout:1187: expect (ObjNum 347 snap 92 seq_num 347) 2024-09-21T09:55:09.611 INFO:tasks.rados.rados.0.smithi033.stdout:1188: finishing write tid 1 to smithi03385152-14 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1188: finishing write tid 2 to smithi03385152-14 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1186: done (4 left) 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1187: done (3 left) 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1191: done (2 left) 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1192: snap_create 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1188: finishing write tid 3 to smithi03385152-14 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1188: finishing write tid 4 to smithi03385152-14 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1188: finishing write tid 5 to smithi03385152-14 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1188: finishing write tid 6 to smithi03385152-14 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 694 (ObjNum 394 snap 107 seq_num 394) dirty exists 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1188: left oid 14 (ObjNum 394 snap 107 seq_num 394) 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1189: finishing write tid 1 to smithi03385152-12 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1189: finishing write tid 2 to smithi03385152-12 2024-09-21T09:55:09.612 INFO:tasks.rados.rados.0.smithi033.stdout:1189: finishing write tid 3 to smithi03385152-12 2024-09-21T09:55:09.613 INFO:tasks.rados.rados.0.smithi033.stdout:1189: finishing write tid 4 to smithi03385152-12 2024-09-21T09:55:09.613 INFO:tasks.rados.rados.0.smithi033.stdout:1189: finishing write tid 5 to smithi03385152-12 2024-09-21T09:55:09.613 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 444 (ObjNum 395 snap 107 seq_num 395) dirty exists 2024-09-21T09:55:09.613 INFO:tasks.rados.rados.0.smithi033.stdout:1189: left oid 12 (ObjNum 395 snap 107 seq_num 395) 2024-09-21T09:55:09.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:09 smithi033 ceph-mon[98546]: pgmap v53: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 966 KiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-21T09:55:09.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:09 smithi033 ceph-mon[98546]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:55:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:09 smithi033 ceph-mon[96568]: pgmap v53: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 966 KiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-21T09:55:09.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:09 smithi033 ceph-mon[96568]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:55:09.768 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:09 smithi105 ceph-mon[86649]: pgmap v53: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 966 KiB/s rd, 1.0 MiB/s wr, 4 op/s 2024-09-21T09:55:09.768 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:09 smithi105 ceph-mon[86649]: osdmap e284: 8 total, 8 up, 8 in 2024-09-21T09:55:10.091 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:55:10.091 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (2m) 16s ago 6m 15.5M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:55:10.091 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (98s) 2s ago 6m 82.4M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:55:10.091 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (8m) 16s ago 8m 15.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:55:10.091 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (7m) 2s ago 7m 19.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:55:10.091 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (60s) 2s ago 11m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (88s) 16s ago 13m 513M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (47s) 16s ago 13m 40.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (7s) 2s ago 11m 22.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (25s) 16s ago 11m 26.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (117s) 16s ago 5m 9663k - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (93s) 2s ago 5m 9915k - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (11m) 16s ago 11m 157M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (10m) 16s ago 10m 167M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:55:10.092 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (10m) 16s ago 10m 145M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:55:10.093 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (9m) 16s ago 9m 94.9M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:55:10.093 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (9m) 2s ago 9m 167M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:55:10.093 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (9m) 2s ago 9m 193M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:55:10.093 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (8m) 2s ago 8m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:55:10.093 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (8m) 2s ago 8m 140M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:55:10.093 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (65s) 2s ago 6m 44.0M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:55:10.291 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:10.449 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:55:10.449 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:55:10.449 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:55:10.449 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:55:10.449 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:55:10.450 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:55:10.450 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:55:10.450 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:55:10.450 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8 2024-09-21T09:55:10.450 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:55:10.450 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:55:10.450 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:55:10.451 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:55:10.451 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:55:10.451 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 10, 2024-09-21T09:55:10.451 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:55:10.451 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:55:10.451 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:55:10.494 INFO:tasks.rados.rados.0.smithi033.stdout:1188: done (2 left) 2024-09-21T09:55:10.494 INFO:tasks.rados.rados.0.smithi033.stdout:1189: done (1 left) 2024-09-21T09:55:10.494 INFO:tasks.rados.rados.0.smithi033.stdout:1192: done (0 left) 2024-09-21T09:55:10.494 INFO:tasks.rados.rados.0.smithi033.stdout:1193: read oid 10 snap -1 2024-09-21T09:55:10.494 INFO:tasks.rados.rados.0.smithi033.stdout:1193: expect (ObjNum 250 snap 65 seq_num 250) 2024-09-21T09:55:10.495 INFO:tasks.rados.rados.0.smithi033.stdout:1194: setattr oid 17 current snap is 108 2024-09-21T09:55:10.495 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 108 2024-09-21T09:55:10.495 INFO:tasks.rados.rados.0.smithi033.stdout:1195: seq_num 397 ranges {2580480=655360,3235840=671744,3907584=532480} 2024-09-21T09:55:10.513 INFO:tasks.rados.rados.0.smithi033.stdout:1195: writing smithi03385152-8 from 2580480 to 3235840 tid 1 2024-09-21T09:55:10.517 INFO:tasks.rados.rados.0.smithi033.stdout:1195: writing smithi03385152-8 from 3235840 to 3907584 tid 2 2024-09-21T09:55:10.520 INFO:tasks.rados.rados.0.smithi033.stdout:1195: writing smithi03385152-8 from 3907584 to 4440064 tid 3 2024-09-21T09:55:10.520 INFO:tasks.rados.rados.0.smithi033.stdout:1196: delete oid 7 current snap is 108 2024-09-21T09:55:10.520 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 446 (ObjNum 338 snap 87 seq_num 338) dirty exists 2024-09-21T09:55:10.520 INFO:tasks.rados.rados.0.smithi033.stdout:1193: expect (ObjNum 250 snap 65 seq_num 250) 2024-09-21T09:55:10.580 INFO:tasks.rados.rados.0.smithi033.stdout:1195: finishing write tid 1 to smithi03385152-8 2024-09-21T09:55:10.580 INFO:tasks.rados.rados.0.smithi033.stdout:1195: finishing write tid 2 to smithi03385152-8 2024-09-21T09:55:10.580 INFO:tasks.rados.rados.0.smithi033.stdout:1193: done (3 left) 2024-09-21T09:55:10.580 INFO:tasks.rados.rados.0.smithi033.stdout:1194: done (2 left) 2024-09-21T09:55:10.580 INFO:tasks.rados.rados.0.smithi033.stdout:1196: done (1 left) 2024-09-21T09:55:10.580 INFO:tasks.rados.rados.0.smithi033.stdout:1197: rollback oid 33 current snap is 108 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 105 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:1195: finishing write tid 3 to smithi03385152-8 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:1195: finishing write tid 4 to smithi03385152-8 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:1195: finishing write tid 5 to smithi03385152-8 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 632 (ObjNum 397 snap 108 seq_num 397) dirty exists 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:1195: left oid 8 (ObjNum 397 snap 108 seq_num 397) 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:1195: done (1 left) 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 108 2024-09-21T09:55:10.581 INFO:tasks.rados.rados.0.smithi033.stdout:1198: seq_num 399 ranges {0=688128,688128=434176,1122304=745472,1867776=24576} 2024-09-21T09:55:10.584 INFO:tasks.rados.rados.0.smithi033.stdout:1198: writing smithi03385152-2 from 0 to 688128 tid 1 2024-09-21T09:55:10.588 INFO:tasks.rados.rados.0.smithi033.stdout:1198: writing smithi03385152-2 from 688128 to 1122304 tid 2 2024-09-21T09:55:10.594 INFO:tasks.rados.rados.0.smithi033.stdout:1198: writing smithi03385152-2 from 1122304 to 1867776 tid 3 2024-09-21T09:55:10.594 INFO:tasks.rados.rados.0.smithi033.stdout:1198: writing smithi03385152-2 from 1867776 to 1892352 tid 4 2024-09-21T09:55:10.594 INFO:tasks.rados.rados.0.smithi033.stdout:1199: read oid 15 snap -1 2024-09-21T09:55:10.595 INFO:tasks.rados.rados.0.smithi033.stdout:1199: expect (ObjNum 361 snap 96 seq_num 361) 2024-09-21T09:55:10.595 INFO:tasks.rados.rados.0.smithi033.stdout:1200: read oid 31 snap 107 2024-09-21T09:55:10.595 INFO:tasks.rados.rados.0.smithi033.stdout:1200: expect (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:55:10.595 INFO:tasks.rados.rados.0.smithi033.stdout:1201: snap_remove snap 103 2024-09-21T09:55:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:10 smithi033 ceph-mon[98546]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:10 smithi033 ceph-mon[98546]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:55:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:10 smithi033 ceph-mon[98546]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:10 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/762846334' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:10 smithi033 ceph-mon[96568]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:10 smithi033 ceph-mon[96568]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:55:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:10 smithi033 ceph-mon[96568]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:10 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/762846334' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:10.771 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:55:10.771 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:55:10.771 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:55:10.771 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:55:10.771 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:55:10.771 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:55:10.771 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:55:10.771 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:55:10.772 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "5/20 daemons upgraded", 2024-09-21T09:55:10.772 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading mon daemons", 2024-09-21T09:55:10.772 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:55:10.772 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:55:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:10 smithi105 ceph-mon[86649]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:10 smithi105 ceph-mon[86649]: osdmap e285: 8 total, 8 up, 8 in 2024-09-21T09:55:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:10 smithi105 ceph-mon[86649]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:10 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/762846334' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:11.504 INFO:tasks.rados.rados.0.smithi033.stdout:1197: finishing rollback tid 0 to smithi03385152-33 2024-09-21T09:55:11.504 INFO:tasks.rados.rados.0.smithi033.stdout:1197: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:55:11.504 INFO:tasks.rados.rados.0.smithi033.stdout:1197: finishing rollback tid 2 to smithi03385152-33 2024-09-21T09:55:11.504 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 443 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:55:11.504 INFO:tasks.rados.rados.0.smithi033.stdout:1198: finishing write tid 1 to smithi03385152-2 2024-09-21T09:55:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:1198: finishing write tid 2 to smithi03385152-2 2024-09-21T09:55:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:1198: finishing write tid 3 to smithi03385152-2 2024-09-21T09:55:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:1198: finishing write tid 4 to smithi03385152-2 2024-09-21T09:55:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:1198: finishing write tid 5 to smithi03385152-2 2024-09-21T09:55:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:1198: finishing write tid 6 to smithi03385152-2 2024-09-21T09:55:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 620 (ObjNum 399 snap 108 seq_num 399) dirty exists 2024-09-21T09:55:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:1198: left oid 2 (ObjNum 399 snap 108 seq_num 399) 2024-09-21T09:55:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:1200: expect (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:55:11.535 INFO:tasks.rados.rados.0.smithi033.stdout:1199: expect (ObjNum 361 snap 96 seq_num 361) 2024-09-21T09:55:11.569 INFO:tasks.rados.rados.0.smithi033.stdout:1197: done (4 left) 2024-09-21T09:55:11.569 INFO:tasks.rados.rados.0.smithi033.stdout:1198: done (3 left) 2024-09-21T09:55:11.569 INFO:tasks.rados.rados.0.smithi033.stdout:1199: done (2 left) 2024-09-21T09:55:11.569 INFO:tasks.rados.rados.0.smithi033.stdout:1200: done (1 left) 2024-09-21T09:55:11.569 INFO:tasks.rados.rados.0.smithi033.stdout:1201: done (0 left) 2024-09-21T09:55:11.569 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 108 2024-09-21T09:55:11.569 INFO:tasks.rados.rados.0.smithi033.stdout:1202: seq_num 400 ranges {1892352=704512,2596864=720896,3317760=483328} 2024-09-21T09:55:11.583 INFO:tasks.rados.rados.0.smithi033.stdout:1202: writing smithi03385152-2 from 1892352 to 2596864 tid 1 2024-09-21T09:55:11.587 INFO:tasks.rados.rados.0.smithi033.stdout:1202: writing smithi03385152-2 from 2596864 to 3317760 tid 2 2024-09-21T09:55:11.590 INFO:tasks.rados.rados.0.smithi033.stdout:1202: writing smithi03385152-2 from 3317760 to 3801088 tid 3 2024-09-21T09:55:11.590 INFO:tasks.rados.rados.0.smithi033.stdout:1203: snap_create 2024-09-21T09:55:11.590 INFO:tasks.rados.rados.0.smithi033.stdout:1202: finishing write tid 1 to smithi03385152-2 2024-09-21T09:55:11.590 INFO:tasks.rados.rados.0.smithi033.stdout:1202: finishing write tid 2 to smithi03385152-2 2024-09-21T09:55:11.592 INFO:tasks.rados.rados.0.smithi033.stdout:1202: finishing write tid 3 to smithi03385152-2 2024-09-21T09:55:11.592 INFO:tasks.rados.rados.0.smithi033.stdout:1202: finishing write tid 4 to smithi03385152-2 2024-09-21T09:55:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:1202: finishing write tid 5 to smithi03385152-2 2024-09-21T09:55:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 624 (ObjNum 400 snap 108 seq_num 400) dirty exists 2024-09-21T09:55:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:1202: left oid 2 (ObjNum 400 snap 108 seq_num 400) 2024-09-21T09:55:11.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-mon[96568]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:11.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-mon[96568]: pgmap v56: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:55:11.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-mon[96568]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:55:11.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-mon[96568]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:11.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:55:11] "GET /metrics HTTP/1.1" 200 36203 "" "Prometheus/2.51.0" 2024-09-21T09:55:11.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-mon[98546]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:11.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-mon[98546]: pgmap v56: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:55:11.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-mon[98546]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:55:11.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:11 smithi033 ceph-mon[98546]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:11 smithi105 ceph-mon[86649]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:11 smithi105 ceph-mon[86649]: pgmap v56: 121 pgs: 121 active+clean; 174 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:55:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:11 smithi105 ceph-mon[86649]: osdmap e286: 8 total, 8 up, 8 in 2024-09-21T09:55:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:11 smithi105 ceph-mon[86649]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:12.532 INFO:tasks.rados.rados.0.smithi033.stdout:1202: done (1 left) 2024-09-21T09:55:12.533 INFO:tasks.rados.rados.0.smithi033.stdout:1203: done (0 left) 2024-09-21T09:55:12.533 INFO:tasks.rados.rados.0.smithi033.stdout:1204: snap_create 2024-09-21T09:55:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:55:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[98546]: Reconfiguring daemon mon.a on smithi033 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:55:12.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:12.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:12.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:12.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:12 smithi033 ceph-mon[96568]: Reconfiguring daemon mon.a on smithi033 2024-09-21T09:55:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: osdmap e287: 8 total, 8 up, 8 in 2024-09-21T09:55:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:12.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: Reconfiguring mon.a (monmap changed)... 2024-09-21T09:55:12.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:12.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:12.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:12.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:12 smithi105 ceph-mon[86649]: Reconfiguring daemon mon.a on smithi033 2024-09-21T09:55:13.437 INFO:tasks.rados.rados.0.smithi033.stdout:1204: done (0 left) 2024-09-21T09:55:13.437 INFO:tasks.rados.rados.0.smithi033.stdout:1205: read oid 17 snap 109 2024-09-21T09:55:13.438 INFO:tasks.rados.rados.0.smithi033.stdout:1205: expect (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:55:13.438 INFO:tasks.rados.rados.0.smithi033.stdout:1206: rmattr oid 32 current snap is 110 2024-09-21T09:55:13.438 INFO:tasks.rados.rados.0.smithi033.stdout:1207: rmattr oid 27 current snap is 110 2024-09-21T09:55:13.438 INFO:tasks.rados.rados.0.smithi033.stdout:1207: done (2 left) 2024-09-21T09:55:13.438 INFO:tasks.rados.rados.0.smithi033.stdout:1208: delete oid 12 current snap is 110 2024-09-21T09:55:13.438 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 0 (ObjNum 129 snap 0 seq_num 91039424) dirty dne 2024-09-21T09:55:13.464 INFO:tasks.rados.rados.0.smithi033.stdout:1205: expect (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:55:13.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[96568]: pgmap v59: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.5 MiB/s wr, 25 op/s 2024-09-21T09:55:13.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[96568]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:55:13.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:13.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:13.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:55:13.732 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:55:13.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:13.733 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[96568]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:55:13.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[98546]: pgmap v59: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.5 MiB/s wr, 25 op/s 2024-09-21T09:55:13.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[98546]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:55:13.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:13.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:13.733 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:55:13.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:55:13.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:13.734 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:13 smithi033 ceph-mon[98546]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:55:13.789 INFO:tasks.rados.rados.0.smithi033.stdout:1205: done (2 left) 2024-09-21T09:55:13.789 INFO:tasks.rados.rados.0.smithi033.stdout:1206: done (1 left) 2024-09-21T09:55:13.789 INFO:tasks.rados.rados.0.smithi033.stdout:1208: done (0 left) 2024-09-21T09:55:13.790 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 110 2024-09-21T09:55:13.790 INFO:tasks.rados.rados.0.smithi033.stdout:1209: seq_num 402 ranges {0=737280,737280=491520,1228800=516096,1744896=196608} 2024-09-21T09:55:13.793 INFO:tasks.rados.rados.0.smithi033.stdout:1209: writing smithi03385152-50 from 0 to 737280 tid 1 2024-09-21T09:55:13.795 INFO:tasks.rados.rados.0.smithi033.stdout:1209: writing smithi03385152-50 from 737280 to 1228800 tid 2 2024-09-21T09:55:13.798 INFO:tasks.rados.rados.0.smithi033.stdout:1209: writing smithi03385152-50 from 1228800 to 1744896 tid 3 2024-09-21T09:55:13.800 INFO:tasks.rados.rados.0.smithi033.stdout:1209: writing smithi03385152-50 from 1744896 to 1941504 tid 4 2024-09-21T09:55:13.800 INFO:tasks.rados.rados.0.smithi033.stdout:1210: snap_remove snap 107 2024-09-21T09:55:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:13 smithi105 ceph-mon[86649]: pgmap v59: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.5 MiB/s wr, 25 op/s 2024-09-21T09:55:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:13 smithi105 ceph-mon[86649]: osdmap e288: 8 total, 8 up, 8 in 2024-09-21T09:55:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:55:14.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:55:14.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:14.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:13 smithi105 ceph-mon[86649]: osdmap e289: 8 total, 8 up, 8 in 2024-09-21T09:55:14.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:14 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:14.546 INFO:tasks.rados.rados.0.smithi033.stdout:1210: done (1 left) 2024-09-21T09:55:14.546 INFO:tasks.rados.rados.0.smithi033.stdout:1211: read oid 37 snap -1 2024-09-21T09:55:14.546 INFO:tasks.rados.rados.0.smithi033.stdout:1211: expect deleted 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1212: copy_from oid 49 from oid 6 current snap is 110 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1209: finishing write tid 1 to smithi03385152-50 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1209: finishing write tid 2 to smithi03385152-50 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1209: finishing write tid 3 to smithi03385152-50 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1209: finishing write tid 4 to smithi03385152-50 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1209: finishing write tid 5 to smithi03385152-50 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1209: finishing write tid 6 to smithi03385152-50 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 629 (ObjNum 402 snap 110 seq_num 402) dirty exists 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1209: left oid 50 (ObjNum 402 snap 110 seq_num 402) 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1209: done (2 left) 2024-09-21T09:55:14.547 INFO:tasks.rados.rados.0.smithi033.stdout:1213: delete oid 44 current snap is 110 2024-09-21T09:55:14.551 INFO:tasks.rados.rados.0.smithi033.stdout:1211: done (2 left) 2024-09-21T09:55:14.551 INFO:tasks.rados.rados.0.smithi033.stdout:1213: done (1 left) 2024-09-21T09:55:14.551 INFO:tasks.rados.rados.0.smithi033.stdout:1214: copy_from oid 24 from oid 29 current snap is 110 2024-09-21T09:55:14.551 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 110 2024-09-21T09:55:14.551 INFO:tasks.rados.rados.0.smithi033.stdout:1215: seq_num 404 ranges {0=770048,770048=712704,1482752=491520} 2024-09-21T09:55:14.554 INFO:tasks.rados.rados.0.smithi033.stdout:1215: writing smithi03385152-1 from 0 to 770048 tid 1 2024-09-21T09:55:14.559 INFO:tasks.rados.rados.0.smithi033.stdout:1215: writing smithi03385152-1 from 770048 to 1482752 tid 2 2024-09-21T09:55:14.562 INFO:tasks.rados.rados.0.smithi033.stdout:1215: writing smithi03385152-1 from 1482752 to 1974272 tid 3 2024-09-21T09:55:14.562 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 110 2024-09-21T09:55:14.562 INFO:tasks.rados.rados.0.smithi033.stdout:1214: finishing copy_from to smithi03385152-24 2024-09-21T09:55:14.562 INFO:tasks.rados.rados.0.smithi033.stdout:1214: got expected ENOENT (src dne) 2024-09-21T09:55:14.562 INFO:tasks.rados.rados.0.smithi033.stdout:1216: seq_num 405 ranges {3153920=786432,3940352=589824,4530176=614400} 2024-09-21T09:55:14.584 INFO:tasks.rados.rados.0.smithi033.stdout:1216: writing smithi03385152-26 from 3153920 to 3940352 tid 1 2024-09-21T09:55:14.587 INFO:tasks.rados.rados.0.smithi033.stdout:1216: writing smithi03385152-26 from 3940352 to 4530176 tid 2 2024-09-21T09:55:14.591 INFO:tasks.rados.rados.0.smithi033.stdout:1216: writing smithi03385152-26 from 4530176 to 5144576 tid 3 2024-09-21T09:55:14.591 INFO:tasks.rados.rados.0.smithi033.stdout:1217: read oid 17 snap -1 2024-09-21T09:55:14.591 INFO:tasks.rados.rados.0.smithi033.stdout:1217: expect (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:55:14.591 INFO:tasks.rados.rados.0.smithi033.stdout:1218: setattr oid 45 current snap is 110 2024-09-21T09:55:14.591 INFO:tasks.rados.rados.0.smithi033.stdout:1214: finishing copy_from racing read to smithi03385152-24 2024-09-21T09:55:14.591 INFO:tasks.rados.rados.0.smithi033.stdout:1212: finishing copy_from racing read to smithi03385152-49 2024-09-21T09:55:14.591 INFO:tasks.rados.rados.0.smithi033.stdout:1212: finishing copy_from to smithi03385152-49 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 433 (ObjNum 347 snap 92 seq_num 347) dirty exists 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1215: finishing write tid 1 to smithi03385152-1 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1215: finishing write tid 2 to smithi03385152-1 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1215: finishing write tid 3 to smithi03385152-1 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1215: finishing write tid 4 to smithi03385152-1 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1215: finishing write tid 5 to smithi03385152-1 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 636 (ObjNum 404 snap 110 seq_num 404) dirty exists 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1215: left oid 1 (ObjNum 404 snap 110 seq_num 404) 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1216: finishing write tid 1 to smithi03385152-26 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1216: finishing write tid 2 to smithi03385152-26 2024-09-21T09:55:14.592 INFO:tasks.rados.rados.0.smithi033.stdout:1212: done (5 left) 2024-09-21T09:55:14.593 INFO:tasks.rados.rados.0.smithi033.stdout:1214: done (4 left) 2024-09-21T09:55:14.593 INFO:tasks.rados.rados.0.smithi033.stdout:1215: done (3 left) 2024-09-21T09:55:14.593 INFO:tasks.rados.rados.0.smithi033.stdout:1219: delete oid 19 current snap is 110 2024-09-21T09:55:14.594 INFO:tasks.rados.rados.0.smithi033.stdout:1216: finishing write tid 3 to smithi03385152-26 2024-09-21T09:55:14.594 INFO:tasks.rados.rados.0.smithi033.stdout:1219: done (3 left) 2024-09-21T09:55:14.594 INFO:tasks.rados.rados.0.smithi033.stdout:1220: delete oid 35 current snap is 110 2024-09-21T09:55:14.595 INFO:tasks.rados.rados.0.smithi033.stdout:1216: finishing write tid 4 to smithi03385152-26 2024-09-21T09:55:14.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 698 (ObjNum 286 snap 70 seq_num 286) dirty exists 2024-09-21T09:55:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:1218: done (3 left) 2024-09-21T09:55:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:1220: done (2 left) 2024-09-21T09:55:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:1221: copy_from oid 36 from oid 24 current snap is 110 2024-09-21T09:55:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 110 2024-09-21T09:55:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:1222: seq_num 408 ranges {0=442368,442368=647168,1089536=507904,1597440=450560} 2024-09-21T09:55:14.599 INFO:tasks.rados.rados.0.smithi033.stdout:1222: writing smithi03385152-39 from 0 to 442368 tid 1 2024-09-21T09:55:14.603 INFO:tasks.rados.rados.0.smithi033.stdout:1222: writing smithi03385152-39 from 442368 to 1089536 tid 2 2024-09-21T09:55:14.606 INFO:tasks.rados.rados.0.smithi033.stdout:1222: writing smithi03385152-39 from 1089536 to 1597440 tid 3 2024-09-21T09:55:14.609 INFO:tasks.rados.rados.0.smithi033.stdout:1222: writing smithi03385152-39 from 1597440 to 2048000 tid 4 2024-09-21T09:55:14.609 INFO:tasks.rados.rados.0.smithi033.stdout:1223: copy_from oid 11 from oid 3 current snap is 110 2024-09-21T09:55:14.609 INFO:tasks.rados.rados.0.smithi033.stdout:1222: finishing write tid 1 to smithi03385152-39 2024-09-21T09:55:14.609 INFO:tasks.rados.rados.0.smithi033.stdout:1222: finishing write tid 2 to smithi03385152-39 2024-09-21T09:55:14.609 INFO:tasks.rados.rados.0.smithi033.stdout:1224: read oid 19 snap -1 2024-09-21T09:55:14.609 INFO:tasks.rados.rados.0.smithi033.stdout:1224: expect deleted 2024-09-21T09:55:14.610 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 110 2024-09-21T09:55:14.610 INFO:tasks.rados.rados.0.smithi033.stdout:1225: seq_num 409 ranges {9158656=458752,9617408=532480,10149888=720896,10870784=352256} 2024-09-21T09:55:14.662 INFO:tasks.rados.rados.0.smithi033.stdout:1225: writing smithi03385152-33 from 9158656 to 9617408 tid 1 2024-09-21T09:55:14.665 INFO:tasks.rados.rados.0.smithi033.stdout:1225: writing smithi03385152-33 from 9617408 to 10149888 tid 2 2024-09-21T09:55:14.669 INFO:tasks.rados.rados.0.smithi033.stdout:1225: writing smithi03385152-33 from 10149888 to 10870784 tid 3 2024-09-21T09:55:14.672 INFO:tasks.rados.rados.0.smithi033.stdout:1225: writing smithi03385152-33 from 10870784 to 11223040 tid 4 2024-09-21T09:55:14.672 INFO:tasks.rados.rados.0.smithi033.stdout:1226: copy_from oid 45 from oid 22 current snap is 110 2024-09-21T09:55:14.672 INFO:tasks.rados.rados.0.smithi033.stdout:1222: finishing write tid 3 to smithi03385152-39 2024-09-21T09:55:14.672 INFO:tasks.rados.rados.0.smithi033.stdout:1224: done (7 left) 2024-09-21T09:55:14.672 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 110 2024-09-21T09:55:14.672 INFO:tasks.rados.rados.0.smithi033.stdout:1222: finishing write tid 4 to smithi03385152-39 2024-09-21T09:55:14.672 INFO:tasks.rados.rados.0.smithi033.stdout:1222: finishing write tid 5 to smithi03385152-39 2024-09-21T09:55:14.672 INFO:tasks.rados.rados.0.smithi033.stdout:1222: finishing write tid 6 to smithi03385152-39 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 155 (ObjNum 408 snap 110 seq_num 408) dirty exists 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:1222: left oid 39 (ObjNum 408 snap 110 seq_num 408) 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:1216: finishing write tid 5 to smithi03385152-26 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 463 (ObjNum 405 snap 110 seq_num 405) dirty exists 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:1216: left oid 26 (ObjNum 405 snap 110 seq_num 405) 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:1223: finishing copy_from racing read to smithi03385152-11 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:1223: finishing copy_from to smithi03385152-11 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 688 (ObjNum 391 snap 106 seq_num 391) dirty exists 2024-09-21T09:55:14.673 INFO:tasks.rados.rados.0.smithi033.stdout:1217: expect (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:55:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:14 smithi033 ceph-mon[96568]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:55:14.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:14 smithi033 ceph-mon[96568]: Reconfiguring daemon mgr.y on smithi033 2024-09-21T09:55:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:14 smithi033 ceph-mon[98546]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:55:14.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:14 smithi033 ceph-mon[98546]: Reconfiguring daemon mgr.y on smithi033 2024-09-21T09:55:14.993 INFO:tasks.rados.rados.0.smithi033.stdout:1221: finishing copy_from racing read to smithi03385152-36 2024-09-21T09:55:14.993 INFO:tasks.rados.rados.0.smithi033.stdout:1221: finishing copy_from to smithi03385152-36 2024-09-21T09:55:14.993 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 157 (ObjNum 285 snap 69 seq_num 285) dirty exists 2024-09-21T09:55:14.993 INFO:tasks.rados.rados.0.smithi033.stdout:1225: finishing write tid 1 to smithi03385152-33 2024-09-21T09:55:14.993 INFO:tasks.rados.rados.0.smithi033.stdout:1225: finishing write tid 2 to smithi03385152-33 2024-09-21T09:55:14.994 INFO:tasks.rados.rados.0.smithi033.stdout:1227: seq_num 410 ranges {1155072=475136,1630208=417792,2048000=475136,2523136=425984,2949120=286720} 2024-09-21T09:55:15.002 INFO:tasks.rados.rados.0.smithi033.stdout:1227: writing smithi03385152-43 from 1155072 to 1630208 tid 1 2024-09-21T09:55:15.004 INFO:tasks.rados.rados.0.smithi033.stdout:1227: writing smithi03385152-43 from 1630208 to 2048000 tid 2 2024-09-21T09:55:15.007 INFO:tasks.rados.rados.0.smithi033.stdout:1227: writing smithi03385152-43 from 2048000 to 2523136 tid 3 2024-09-21T09:55:15.009 INFO:tasks.rados.rados.0.smithi033.stdout:1227: writing smithi03385152-43 from 2523136 to 2949120 tid 4 2024-09-21T09:55:15.011 INFO:tasks.rados.rados.0.smithi033.stdout:1227: writing smithi03385152-43 from 2949120 to 3235840 tid 5 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1216: done (7 left) 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1217: done (6 left) 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1221: done (5 left) 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1222: done (4 left) 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1223: done (3 left) 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 110 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1225: finishing write tid 3 to smithi03385152-33 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1225: finishing write tid 4 to smithi03385152-33 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1225: finishing write tid 5 to smithi03385152-33 2024-09-21T09:55:15.012 INFO:tasks.rados.rados.0.smithi033.stdout:1228: seq_num 411 ranges {0=491520,491520=753664,1245184=688128,1933312=163840} 2024-09-21T09:55:15.014 INFO:tasks.rados.rados.0.smithi033.stdout:1228: writing smithi03385152-12 from 0 to 491520 tid 1 2024-09-21T09:55:15.019 INFO:tasks.rados.rados.0.smithi033.stdout:1228: writing smithi03385152-12 from 491520 to 1245184 tid 2 2024-09-21T09:55:15.023 INFO:tasks.rados.rados.0.smithi033.stdout:1228: writing smithi03385152-12 from 1245184 to 1933312 tid 3 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1228: writing smithi03385152-12 from 1933312 to 2097152 tid 4 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1229: read oid 30 snap -1 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1229: expect (ObjNum 387 snap 105 seq_num 387) 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1230: rmattr oid 46 current snap is 110 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1230: done (5 left) 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1231: read oid 5 snap -1 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1231: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1232: copy_from oid 8 from oid 17 current snap is 110 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1225: finishing write tid 6 to smithi03385152-33 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 448 (ObjNum 409 snap 110 seq_num 409) dirty exists 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1225: left oid 33 (ObjNum 409 snap 110 seq_num 409) 2024-09-21T09:55:15.025 INFO:tasks.rados.rados.0.smithi033.stdout:1226: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:55:15.026 INFO:tasks.rados.rados.0.smithi033.stdout:1225: done (6 left) 2024-09-21T09:55:15.026 INFO:tasks.rados.rados.0.smithi033.stdout:1233: read oid 6 snap -1 2024-09-21T09:55:15.026 INFO:tasks.rados.rados.0.smithi033.stdout:1233: expect (ObjNum 347 snap 92 seq_num 347) 2024-09-21T09:55:15.026 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 110 2024-09-21T09:55:15.026 INFO:tasks.rados.rados.0.smithi033.stdout:1226: finishing copy_from to smithi03385152-45 2024-09-21T09:55:15.026 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 642 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:55:15.026 INFO:tasks.rados.rados.0.smithi033.stdout:1234: seq_num 412 ranges {0=507904,507904=630784,1138688=491520,1630208=483328} 2024-09-21T09:55:15.027 INFO:tasks.rados.rados.0.smithi033.stdout:1234: writing smithi03385152-44 from 0 to 507904 tid 1 2024-09-21T09:55:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:14 smithi105 ceph-mon[86649]: Reconfiguring mgr.y (monmap changed)... 2024-09-21T09:55:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:14 smithi105 ceph-mon[86649]: Reconfiguring daemon mgr.y on smithi033 2024-09-21T09:55:15.031 INFO:tasks.rados.rados.0.smithi033.stdout:1234: writing smithi03385152-44 from 507904 to 1138688 tid 2 2024-09-21T09:55:15.034 INFO:tasks.rados.rados.0.smithi033.stdout:1234: writing smithi03385152-44 from 1138688 to 1630208 tid 3 2024-09-21T09:55:15.037 INFO:tasks.rados.rados.0.smithi033.stdout:1234: writing smithi03385152-44 from 1630208 to 2113536 tid 4 2024-09-21T09:55:15.037 INFO:tasks.rados.rados.0.smithi033.stdout:1226: done (7 left) 2024-09-21T09:55:15.037 INFO:tasks.rados.rados.0.smithi033.stdout:1235: rollback oid 49 current snap is 110 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1227: finishing write tid 1 to smithi03385152-43 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1227: finishing write tid 2 to smithi03385152-43 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1227: finishing write tid 3 to smithi03385152-43 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1227: finishing write tid 4 to smithi03385152-43 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1227: finishing write tid 5 to smithi03385152-43 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1227: finishing write tid 6 to smithi03385152-43 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1227: finishing write tid 7 to smithi03385152-43 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 694 (ObjNum 410 snap 110 seq_num 410) dirty exists 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1227: left oid 43 (ObjNum 410 snap 110 seq_num 410) 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1228: finishing write tid 1 to smithi03385152-12 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1228: finishing write tid 2 to smithi03385152-12 2024-09-21T09:55:15.038 INFO:tasks.rados.rados.0.smithi033.stdout:1231: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:55:15.042 INFO:tasks.rados.rados.0.smithi033.stdout:1228: finishing write tid 3 to smithi03385152-12 2024-09-21T09:55:15.042 INFO:tasks.rados.rados.0.smithi033.stdout:1233: expect (ObjNum 347 snap 92 seq_num 347) 2024-09-21T09:55:15.076 INFO:tasks.rados.rados.0.smithi033.stdout:1228: finishing write tid 4 to smithi03385152-12 2024-09-21T09:55:15.076 INFO:tasks.rados.rados.0.smithi033.stdout:1228: finishing write tid 5 to smithi03385152-12 2024-09-21T09:55:15.076 INFO:tasks.rados.rados.0.smithi033.stdout:1228: finishing write tid 6 to smithi03385152-12 2024-09-21T09:55:15.076 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 468 (ObjNum 411 snap 110 seq_num 411) dirty exists 2024-09-21T09:55:15.076 INFO:tasks.rados.rados.0.smithi033.stdout:1228: left oid 12 (ObjNum 411 snap 110 seq_num 411) 2024-09-21T09:55:15.076 INFO:tasks.rados.rados.0.smithi033.stdout:1234: finishing write tid 1 to smithi03385152-44 2024-09-21T09:55:15.076 INFO:tasks.rados.rados.0.smithi033.stdout:1234: finishing write tid 2 to smithi03385152-44 2024-09-21T09:55:15.076 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 49 to 98 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:1234: finishing write tid 3 to smithi03385152-44 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:1234: finishing write tid 4 to smithi03385152-44 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:1234: finishing write tid 5 to smithi03385152-44 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:1234: finishing write tid 6 to smithi03385152-44 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 633 (ObjNum 412 snap 110 seq_num 412) dirty exists 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:1234: left oid 44 (ObjNum 412 snap 110 seq_num 412) 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:1232: finishing copy_from racing read to smithi03385152-8 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:1232: finishing copy_from to smithi03385152-8 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 446 (ObjNum 338 snap 87 seq_num 338) dirty exists 2024-09-21T09:55:15.077 INFO:tasks.rados.rados.0.smithi033.stdout:1229: expect (ObjNum 387 snap 105 seq_num 387) 2024-09-21T09:55:15.481 INFO:tasks.rados.rados.0.smithi033.stdout:1235: finishing rollback tid 0 to smithi03385152-49 2024-09-21T09:55:15.481 INFO:tasks.rados.rados.0.smithi033.stdout:1235: finishing rollback tid 1 to smithi03385152-49 2024-09-21T09:55:15.481 INFO:tasks.rados.rados.0.smithi033.stdout:1235: finishing rollback tid 2 to smithi03385152-49 2024-09-21T09:55:15.481 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 471 (ObjNum 347 snap 92 seq_num 347) dirty exists 2024-09-21T09:55:15.481 INFO:tasks.rados.rados.0.smithi033.stdout:1227: done (7 left) 2024-09-21T09:55:15.482 INFO:tasks.rados.rados.0.smithi033.stdout:1228: done (6 left) 2024-09-21T09:55:15.482 INFO:tasks.rados.rados.0.smithi033.stdout:1229: done (5 left) 2024-09-21T09:55:15.482 INFO:tasks.rados.rados.0.smithi033.stdout:1231: done (4 left) 2024-09-21T09:55:15.482 INFO:tasks.rados.rados.0.smithi033.stdout:1232: done (3 left) 2024-09-21T09:55:15.482 INFO:tasks.rados.rados.0.smithi033.stdout:1233: done (2 left) 2024-09-21T09:55:15.482 INFO:tasks.rados.rados.0.smithi033.stdout:1234: done (1 left) 2024-09-21T09:55:15.483 INFO:tasks.rados.rados.0.smithi033.stdout:1235: done (0 left) 2024-09-21T09:55:15.483 INFO:tasks.rados.rados.0.smithi033.stdout:1236: read oid 7 snap -1 2024-09-21T09:55:15.483 INFO:tasks.rados.rados.0.smithi033.stdout:1236: expect deleted 2024-09-21T09:55:15.483 INFO:tasks.rados.rados.0.smithi033.stdout:1237: rollback oid 20 current snap is 110 2024-09-21T09:55:15.483 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 108 2024-09-21T09:55:15.483 INFO:tasks.rados.rados.0.smithi033.stdout:1238: read oid 15 snap -1 2024-09-21T09:55:15.483 INFO:tasks.rados.rados.0.smithi033.stdout:1238: expect (ObjNum 361 snap 96 seq_num 361) 2024-09-21T09:55:15.484 INFO:tasks.rados.rados.0.smithi033.stdout:1239: delete oid 29 current snap is 110 2024-09-21T09:55:15.485 INFO:tasks.rados.rados.0.smithi033.stdout:1236: done (3 left) 2024-09-21T09:55:15.485 INFO:tasks.rados.rados.0.smithi033.stdout:1239: done (2 left) 2024-09-21T09:55:15.485 INFO:tasks.rados.rados.0.smithi033.stdout:1240: rollback oid 11 current snap is 110 2024-09-21T09:55:15.485 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 11 to 98 2024-09-21T09:55:15.485 INFO:tasks.rados.rados.0.smithi033.stdout:1241: read oid 1 snap 108 2024-09-21T09:55:15.485 INFO:tasks.rados.rados.0.smithi033.stdout:1241: expect deleted 2024-09-21T09:55:15.486 INFO:tasks.rados.rados.0.smithi033.stdout:1242: read oid 32 snap -1 2024-09-21T09:55:15.486 INFO:tasks.rados.rados.0.smithi033.stdout:1242: expect deleted 2024-09-21T09:55:15.486 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 110 2024-09-21T09:55:15.486 INFO:tasks.rados.rados.0.smithi033.stdout:1243: seq_num 414 ranges {9158656=540672,9699328=802816,10502144=458752,10960896=344064} 2024-09-21T09:55:15.536 INFO:tasks.rados.rados.0.smithi033.stdout:1243: writing smithi03385152-24 from 9158656 to 9699328 tid 1 2024-09-21T09:55:15.541 INFO:tasks.rados.rados.0.smithi033.stdout:1243: writing smithi03385152-24 from 9699328 to 10502144 tid 2 2024-09-21T09:55:15.544 INFO:tasks.rados.rados.0.smithi033.stdout:1243: writing smithi03385152-24 from 10502144 to 10960896 tid 3 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1243: writing smithi03385152-24 from 10960896 to 11304960 tid 4 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1244: rmattr oid 26 current snap is 110 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1237: finishing rollback tid 0 to smithi03385152-20 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1245: rmattr oid 45 current snap is 110 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1245: done (7 left) 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1246: snap_create 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1237: finishing rollback tid 1 to smithi03385152-20 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1237: finishing rollback tid 2 to smithi03385152-20 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 684 (ObjNum 392 snap 106 seq_num 392) dirty exists 2024-09-21T09:55:15.547 INFO:tasks.rados.rados.0.smithi033.stdout:1238: expect (ObjNum 361 snap 96 seq_num 361) 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1240: finishing rollback tid 0 to smithi03385152-11 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1240: finishing rollback tid 1 to smithi03385152-11 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1240: finishing rollback tid 2 to smithi03385152-11 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 697 (ObjNum 356 snap 94 seq_num 356) dirty exists 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1243: finishing write tid 1 to smithi03385152-24 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1243: finishing write tid 2 to smithi03385152-24 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1243: finishing write tid 3 to smithi03385152-24 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 472 (ObjNum 405 snap 110 seq_num 405) dirty exists 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1243: finishing write tid 4 to smithi03385152-24 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1243: finishing write tid 5 to smithi03385152-24 2024-09-21T09:55:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:1243: finishing write tid 6 to smithi03385152-24 2024-09-21T09:55:15.582 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 637 (ObjNum 414 snap 110 seq_num 414) dirty exists 2024-09-21T09:55:15.582 INFO:tasks.rados.rados.0.smithi033.stdout:1243: left oid 24 (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: pgmap v62: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.5 MiB/s wr, 25 op/s 2024-09-21T09:55:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:55:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:55:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[98546]: Reconfiguring daemon mon.c on smithi033 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: pgmap v62: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.5 MiB/s wr, 25 op/s 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:15 smithi033 ceph-mon[96568]: Reconfiguring daemon mon.c on smithi033 2024-09-21T09:55:16.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:16.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:16.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: pgmap v62: 121 pgs: 121 active+clean; 186 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 4.9 MiB/s rd, 5.5 MiB/s wr, 25 op/s 2024-09-21T09:55:16.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: osdmap e290: 8 total, 8 up, 8 in 2024-09-21T09:55:16.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:16.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:16.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: Reconfiguring mon.c (monmap changed)... 2024-09-21T09:55:16.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:16.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:16.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:16.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:15 smithi105 ceph-mon[86649]: Reconfiguring daemon mon.c on smithi033 2024-09-21T09:55:16.560 INFO:tasks.rados.rados.0.smithi033.stdout:1237: done (7 left) 2024-09-21T09:55:16.560 INFO:tasks.rados.rados.0.smithi033.stdout:1238: done (6 left) 2024-09-21T09:55:16.560 INFO:tasks.rados.rados.0.smithi033.stdout:1240: done (5 left) 2024-09-21T09:55:16.560 INFO:tasks.rados.rados.0.smithi033.stdout:1241: done (4 left) 2024-09-21T09:55:16.560 INFO:tasks.rados.rados.0.smithi033.stdout:1242: done (3 left) 2024-09-21T09:55:16.560 INFO:tasks.rados.rados.0.smithi033.stdout:1243: done (2 left) 2024-09-21T09:55:16.560 INFO:tasks.rados.rados.0.smithi033.stdout:1244: done (1 left) 2024-09-21T09:55:16.560 INFO:tasks.rados.rados.0.smithi033.stdout:1246: done (0 left) 2024-09-21T09:55:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:1247: rollback oid 39 current snap is 111 2024-09-21T09:55:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 39 to 101 2024-09-21T09:55:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 111 2024-09-21T09:55:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:1248: seq_num 415 ranges {2113536=557056,2670592=688128,3358720=663552,4022272=253952} 2024-09-21T09:55:16.574 INFO:tasks.rados.rados.0.smithi033.stdout:1248: writing smithi03385152-44 from 2113536 to 2670592 tid 1 2024-09-21T09:55:16.578 INFO:tasks.rados.rados.0.smithi033.stdout:1248: writing smithi03385152-44 from 2670592 to 3358720 tid 2 2024-09-21T09:55:16.582 INFO:tasks.rados.rados.0.smithi033.stdout:1248: writing smithi03385152-44 from 3358720 to 4022272 tid 3 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1248: writing smithi03385152-44 from 4022272 to 4276224 tid 4 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1249: read oid 16 snap 110 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1249: expect (ObjNum 383 snap 103 seq_num 383) 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1250: read oid 17 snap -1 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1250: expect (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1251: read oid 34 snap 98 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1251: expect (ObjNum 313 snap 83 seq_num 313) 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1252: delete oid 11 current snap is 111 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1247: finishing rollback tid 0 to smithi03385152-39 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:1247: finishing rollback tid 1 to smithi03385152-39 2024-09-21T09:55:16.585 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 158 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:55:16.586 INFO:tasks.rados.rados.0.smithi033.stdout:1248: finishing write tid 1 to smithi03385152-44 2024-09-21T09:55:16.586 INFO:tasks.rados.rados.0.smithi033.stdout:1248: finishing write tid 2 to smithi03385152-44 2024-09-21T09:55:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:1248: finishing write tid 3 to smithi03385152-44 2024-09-21T09:55:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:1248: finishing write tid 4 to smithi03385152-44 2024-09-21T09:55:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:1248: finishing write tid 5 to smithi03385152-44 2024-09-21T09:55:16.590 INFO:tasks.rados.rados.0.smithi033.stdout:1248: finishing write tid 6 to smithi03385152-44 2024-09-21T09:55:16.590 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 638 (ObjNum 415 snap 111 seq_num 415) dirty exists 2024-09-21T09:55:16.590 INFO:tasks.rados.rados.0.smithi033.stdout:1248: left oid 44 (ObjNum 415 snap 111 seq_num 415) 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:1247: done (5 left) 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:1248: done (4 left) 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:1252: done (3 left) 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:1253: rollback oid 31 current snap is 111 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 31 to 106 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:1254: read oid 30 snap -1 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:1254: expect (ObjNum 387 snap 105 seq_num 387) 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:1255: copy_from oid 43 from oid 25 current snap is 111 2024-09-21T09:55:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:1256: snap_remove snap 108 2024-09-21T09:55:16.843 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:16.843 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:16.843 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:16.843 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:55:16.843 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:16.843 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:16.844 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:16.844 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:16.844 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:55:16.844 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:16.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:16.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:16.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:16.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:55:16.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:17.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:16 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:16.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:17.557 INFO:tasks.rados.rados.0.smithi033.stdout:1256: done (6 left) 2024-09-21T09:55:17.557 INFO:tasks.rados.rados.0.smithi033.stdout:1257: snap_remove snap 105 2024-09-21T09:55:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:17 smithi033 ceph-mon[98546]: pgmap v64: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 26 op/s 2024-09-21T09:55:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:17 smithi033 ceph-mon[98546]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:55:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:17 smithi033 ceph-mon[98546]: Reconfiguring daemon osd.0 on smithi033 2024-09-21T09:55:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:17 smithi033 ceph-mon[98546]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:55:17.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:17 smithi033 ceph-mon[96568]: pgmap v64: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 26 op/s 2024-09-21T09:55:17.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:17 smithi033 ceph-mon[96568]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:55:17.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:17 smithi033 ceph-mon[96568]: Reconfiguring daemon osd.0 on smithi033 2024-09-21T09:55:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:17 smithi033 ceph-mon[96568]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:55:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:17 smithi105 ceph-mon[86649]: pgmap v64: 121 pgs: 121 active+clean; 226 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 26 op/s 2024-09-21T09:55:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:17 smithi105 ceph-mon[86649]: Reconfiguring osd.0 (monmap changed)... 2024-09-21T09:55:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:17 smithi105 ceph-mon[86649]: Reconfiguring daemon osd.0 on smithi033 2024-09-21T09:55:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:17 smithi105 ceph-mon[86649]: osdmap e291: 8 total, 8 up, 8 in 2024-09-21T09:55:18.559 INFO:tasks.rados.rados.0.smithi033.stdout:1257: done (6 left) 2024-09-21T09:55:18.559 INFO:tasks.rados.rados.0.smithi033.stdout:1258: copy_from oid 38 from oid 46 current snap is 111 2024-09-21T09:55:18.559 INFO:tasks.rados.rados.0.smithi033.stdout:1255: finishing copy_from to smithi03385152-43 2024-09-21T09:55:18.559 INFO:tasks.rados.rados.0.smithi033.stdout:1255: got expected ENOENT (src dne) 2024-09-21T09:55:18.560 INFO:tasks.rados.rados.0.smithi033.stdout:1255: finishing copy_from racing read to smithi03385152-43 2024-09-21T09:55:18.560 INFO:tasks.rados.rados.0.smithi033.stdout:1251: expect (ObjNum 313 snap 83 seq_num 313) 2024-09-21T09:55:18.584 INFO:tasks.rados.rados.0.smithi033.stdout:1250: expect (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:55:18.904 INFO:tasks.rados.rados.0.smithi033.stdout:1249: expect (ObjNum 383 snap 103 seq_num 383) 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[98546]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[98546]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[98546]: Reconfiguring daemon osd.1 on smithi033 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[96568]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:18.964 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[96568]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:55:18.965 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:55:18.965 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:18.965 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:18 smithi033 ceph-mon[96568]: Reconfiguring daemon osd.1 on smithi033 2024-09-21T09:55:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:18 smithi105 ceph-mon[86649]: osdmap e292: 8 total, 8 up, 8 in 2024-09-21T09:55:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:18 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:18 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:18 smithi105 ceph-mon[86649]: Reconfiguring osd.1 (monmap changed)... 2024-09-21T09:55:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:18 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:55:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:18 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:18 smithi105 ceph-mon[86649]: Reconfiguring daemon osd.1 on smithi033 2024-09-21T09:55:19.602 INFO:tasks.rados.rados.0.smithi033.stdout:1253: finishing rollback tid 0 to smithi03385152-31 2024-09-21T09:55:19.602 INFO:tasks.rados.rados.0.smithi033.stdout:1253: finishing rollback tid 1 to smithi03385152-31 2024-09-21T09:55:19.602 INFO:tasks.rados.rados.0.smithi033.stdout:1253: finishing rollback tid 2 to smithi03385152-31 2024-09-21T09:55:19.602 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 791 (ObjNum 344 snap 91 seq_num 344) dirty exists 2024-09-21T09:55:19.602 INFO:tasks.rados.rados.0.smithi033.stdout:1254: expect (ObjNum 387 snap 105 seq_num 387) 2024-09-21T09:55:19.832 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[98546]: pgmap v67: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.5 MiB/s wr, 27 op/s 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[98546]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[96568]: pgmap v67: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.5 MiB/s wr, 27 op/s 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[96568]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:55:19.833 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:19 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:20.001 INFO:tasks.rados.rados.0.smithi033.stdout:1258: finishing copy_from racing read to smithi03385152-38 2024-09-21T09:55:20.001 INFO:tasks.rados.rados.0.smithi033.stdout:1258: finishing copy_from to smithi03385152-38 2024-09-21T09:55:20.001 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 671 (ObjNum 344 snap 91 seq_num 344) dirty exists 2024-09-21T09:55:20.001 INFO:tasks.rados.rados.0.smithi033.stdout:1249: done (6 left) 2024-09-21T09:55:20.002 INFO:tasks.rados.rados.0.smithi033.stdout:1250: done (5 left) 2024-09-21T09:55:20.003 INFO:tasks.rados.rados.0.smithi033.stdout:1251: done (4 left) 2024-09-21T09:55:20.003 INFO:tasks.rados.rados.0.smithi033.stdout:1253: done (3 left) 2024-09-21T09:55:20.003 INFO:tasks.rados.rados.0.smithi033.stdout:1254: done (2 left) 2024-09-21T09:55:20.003 INFO:tasks.rados.rados.0.smithi033.stdout:1255: done (1 left) 2024-09-21T09:55:20.003 INFO:tasks.rados.rados.0.smithi033.stdout:1258: done (0 left) 2024-09-21T09:55:20.003 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 111 2024-09-21T09:55:20.003 INFO:tasks.rados.rados.0.smithi033.stdout:1259: seq_num 417 ranges {966656=589824,1556480=458752,2015232=679936,2695168=466944} 2024-09-21T09:55:20.011 INFO:tasks.rados.rados.0.smithi033.stdout:1259: writing smithi03385152-38 from 966656 to 1556480 tid 1 2024-09-21T09:55:20.013 INFO:tasks.rados.rados.0.smithi033.stdout:1259: writing smithi03385152-38 from 1556480 to 2015232 tid 2 2024-09-21T09:55:20.017 INFO:tasks.rados.rados.0.smithi033.stdout:1259: writing smithi03385152-38 from 2015232 to 2695168 tid 3 2024-09-21T09:55:20.020 INFO:tasks.rados.rados.0.smithi033.stdout:1259: writing smithi03385152-38 from 2695168 to 3162112 tid 4 2024-09-21T09:55:20.020 INFO:tasks.rados.rados.0.smithi033.stdout:1259: finishing write tid 1 to smithi03385152-38 2024-09-21T09:55:20.020 INFO:tasks.rados.rados.0.smithi033.stdout:1259: finishing write tid 2 to smithi03385152-38 2024-09-21T09:55:20.020 INFO:tasks.rados.rados.0.smithi033.stdout:1259: finishing write tid 3 to smithi03385152-38 2024-09-21T09:55:20.020 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 111 2024-09-21T09:55:20.020 INFO:tasks.rados.rados.0.smithi033.stdout:1260: seq_num 418 ranges {450560=606208,1056768=737280,1794048=434176,2228224=434176} 2024-09-21T09:55:20.026 INFO:tasks.rados.rados.0.smithi033.stdout:1260: writing smithi03385152-34 from 450560 to 1056768 tid 1 2024-09-21T09:55:20.030 INFO:tasks.rados.rados.0.smithi033.stdout:1260: writing smithi03385152-34 from 1056768 to 1794048 tid 2 2024-09-21T09:55:20.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:19 smithi105 ceph-mon[86649]: pgmap v67: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 8.5 MiB/s wr, 27 op/s 2024-09-21T09:55:20.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:19 smithi105 ceph-mon[86649]: osdmap e293: 8 total, 8 up, 8 in 2024-09-21T09:55:20.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:19 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:20.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:19 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:20.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:19 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:55:20.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:19 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:20.032 INFO:tasks.rados.rados.0.smithi033.stdout:1260: writing smithi03385152-34 from 1794048 to 2228224 tid 3 2024-09-21T09:55:20.037 INFO:tasks.rados.rados.0.smithi033.stdout:1260: writing smithi03385152-34 from 2228224 to 2662400 tid 4 2024-09-21T09:55:20.037 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 111 2024-09-21T09:55:20.037 INFO:tasks.rados.rados.0.smithi033.stdout:1259: finishing write tid 4 to smithi03385152-38 2024-09-21T09:55:20.037 INFO:tasks.rados.rados.0.smithi033.stdout:1259: finishing write tid 5 to smithi03385152-38 2024-09-21T09:55:20.037 INFO:tasks.rados.rados.0.smithi033.stdout:1259: finishing write tid 6 to smithi03385152-38 2024-09-21T09:55:20.037 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 676 (ObjNum 417 snap 111 seq_num 417) dirty exists 2024-09-21T09:55:20.037 INFO:tasks.rados.rados.0.smithi033.stdout:1259: left oid 38 (ObjNum 417 snap 111 seq_num 417) 2024-09-21T09:55:20.038 INFO:tasks.rados.rados.0.smithi033.stdout:1260: finishing write tid 1 to smithi03385152-34 2024-09-21T09:55:20.038 INFO:tasks.rados.rados.0.smithi033.stdout:1261: seq_num 419 ranges {5505024=622592,6127616=671744,6799360=647168,7446528=286720} 2024-09-21T09:55:20.068 INFO:tasks.rados.rados.0.smithi033.stdout:1261: writing smithi03385152-22 from 5505024 to 6127616 tid 1 2024-09-21T09:55:20.071 INFO:tasks.rados.rados.0.smithi033.stdout:1261: writing smithi03385152-22 from 6127616 to 6799360 tid 2 2024-09-21T09:55:20.075 INFO:tasks.rados.rados.0.smithi033.stdout:1261: writing smithi03385152-22 from 6799360 to 7446528 tid 3 2024-09-21T09:55:20.077 INFO:tasks.rados.rados.0.smithi033.stdout:1261: writing smithi03385152-22 from 7446528 to 7733248 tid 4 2024-09-21T09:55:20.077 INFO:tasks.rados.rados.0.smithi033.stdout:1259: done (2 left) 2024-09-21T09:55:20.077 INFO:tasks.rados.rados.0.smithi033.stdout:1262: rmattr oid 15 current snap is 111 2024-09-21T09:55:20.077 INFO:tasks.rados.rados.0.smithi033.stdout:1262: done (2 left) 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1263: delete oid 20 current snap is 111 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1260: finishing write tid 2 to smithi03385152-34 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1260: finishing write tid 3 to smithi03385152-34 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1260: finishing write tid 4 to smithi03385152-34 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1260: finishing write tid 5 to smithi03385152-34 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1260: finishing write tid 6 to smithi03385152-34 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 711 (ObjNum 418 snap 111 seq_num 418) dirty exists 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1260: left oid 34 (ObjNum 418 snap 111 seq_num 418) 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1261: finishing write tid 1 to smithi03385152-22 2024-09-21T09:55:20.078 INFO:tasks.rados.rados.0.smithi033.stdout:1261: finishing write tid 2 to smithi03385152-22 2024-09-21T09:55:20.080 INFO:tasks.rados.rados.0.smithi033.stdout:1261: finishing write tid 3 to smithi03385152-22 2024-09-21T09:55:20.080 INFO:tasks.rados.rados.0.smithi033.stdout:1260: done (2 left) 2024-09-21T09:55:20.080 INFO:tasks.rados.rados.0.smithi033.stdout:1263: done (1 left) 2024-09-21T09:55:20.080 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 111 2024-09-21T09:55:20.080 INFO:tasks.rados.rados.0.smithi033.stdout:1264: seq_num 421 ranges {0=655360,655360=442368,1097728=663552,1761280=499712} 2024-09-21T09:55:20.083 INFO:tasks.rados.rados.0.smithi033.stdout:1264: writing smithi03385152-5 from 0 to 655360 tid 1 2024-09-21T09:55:20.085 INFO:tasks.rados.rados.0.smithi033.stdout:1264: writing smithi03385152-5 from 655360 to 1097728 tid 2 2024-09-21T09:55:20.089 INFO:tasks.rados.rados.0.smithi033.stdout:1264: writing smithi03385152-5 from 1097728 to 1761280 tid 3 2024-09-21T09:55:20.092 INFO:tasks.rados.rados.0.smithi033.stdout:1264: writing smithi03385152-5 from 1761280 to 2260992 tid 4 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:1265: delete oid 38 current snap is 111 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:1261: finishing write tid 4 to smithi03385152-22 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:1261: finishing write tid 5 to smithi03385152-22 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:1261: finishing write tid 6 to smithi03385152-22 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 717 (ObjNum 419 snap 111 seq_num 419) dirty exists 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:1261: left oid 22 (ObjNum 419 snap 111 seq_num 419) 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:1264: finishing write tid 1 to smithi03385152-5 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:1264: finishing write tid 2 to smithi03385152-5 2024-09-21T09:55:20.093 INFO:tasks.rados.rados.0.smithi033.stdout:1264: finishing write tid 3 to smithi03385152-5 2024-09-21T09:55:20.094 INFO:tasks.rados.rados.0.smithi033.stdout:1261: done (2 left) 2024-09-21T09:55:20.094 INFO:tasks.rados.rados.0.smithi033.stdout:1265: done (1 left) 2024-09-21T09:55:20.094 INFO:tasks.rados.rados.0.smithi033.stdout:1266: setattr oid 17 current snap is 111 2024-09-21T09:55:20.095 INFO:tasks.rados.rados.0.smithi033.stdout:1264: finishing write tid 4 to smithi03385152-5 2024-09-21T09:55:20.095 INFO:tasks.rados.rados.0.smithi033.stdout:1264: finishing write tid 5 to smithi03385152-5 2024-09-21T09:55:20.096 INFO:tasks.rados.rados.0.smithi033.stdout:1267: read oid 21 snap -1 2024-09-21T09:55:20.096 INFO:tasks.rados.rados.0.smithi033.stdout:1267: expect (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:1268: rmattr oid 27 current snap is 111 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:1268: done (3 left) 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:1269: copy_from oid 19 from oid 2 current snap is 111 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:1270: read oid 37 snap -1 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:1270: expect deleted 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:1271: rollback oid 3 current snap is 111 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 3 to 101 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:1272: rollback oid 35 current snap is 111 2024-09-21T09:55:20.097 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 35 to 110 2024-09-21T09:55:20.098 INFO:tasks.rados.rados.0.smithi033.stdout:1273: snap_create 2024-09-21T09:55:20.099 INFO:tasks.rados.rados.0.smithi033.stdout:1264: finishing write tid 6 to smithi03385152-5 2024-09-21T09:55:20.099 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 164 (ObjNum 421 snap 111 seq_num 421) dirty exists 2024-09-21T09:55:20.099 INFO:tasks.rados.rados.0.smithi033.stdout:1264: left oid 5 (ObjNum 421 snap 111 seq_num 421) 2024-09-21T09:55:20.099 INFO:tasks.rados.rados.0.smithi033.stdout:1271: finishing rollback tid 0 to smithi03385152-3 2024-09-21T09:55:20.099 INFO:tasks.rados.rados.0.smithi033.stdout:1271: finishing rollback tid 1 to smithi03385152-3 2024-09-21T09:55:20.099 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 720 (ObjNum 33 snap 0 seq_num 1498948933) dirty dne 2024-09-21T09:55:20.100 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 488 (ObjNum 338 snap 87 seq_num 338) dirty exists 2024-09-21T09:55:20.101 INFO:tasks.rados.rados.0.smithi033.stdout:1272: finishing rollback tid 1 to smithi03385152-35 2024-09-21T09:55:20.101 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 784 (ObjNum 177 snap 0 seq_num 1163894144) dirty dne 2024-09-21T09:55:20.110 INFO:tasks.rados.rados.0.smithi033.stdout:1267: expect (ObjNum 308 snap 81 seq_num 308) 2024-09-21T09:55:20.456 INFO:tasks.rados.rados.0.smithi033.stdout:1269: finishing copy_from racing read to smithi03385152-19 2024-09-21T09:55:20.456 INFO:tasks.rados.rados.0.smithi033.stdout:1269: finishing copy_from to smithi03385152-19 2024-09-21T09:55:20.456 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 624 (ObjNum 400 snap 108 seq_num 400) dirty exists 2024-09-21T09:55:20.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:20.570 INFO:tasks.rados.rados.0.smithi033.stdout:1264: done (7 left) 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:1266: done (6 left) 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:1267: done (5 left) 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:1269: done (4 left) 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:1270: done (3 left) 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:1271: done (2 left) 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:1272: done (1 left) 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:1273: done (0 left) 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 112 2024-09-21T09:55:20.571 INFO:tasks.rados.rados.0.smithi033.stdout:1274: seq_num 423 ranges {2260992=688128,2949120=614400,3563520=622592,4186112=368640} 2024-09-21T09:55:20.591 INFO:tasks.rados.rados.0.smithi033.stdout:1274: writing smithi03385152-5 from 2260992 to 2949120 tid 1 2024-09-21T09:55:20.591 INFO:tasks.rados.rados.0.smithi033.stdout:1274: writing smithi03385152-5 from 2949120 to 3563520 tid 2 2024-09-21T09:55:20.593 INFO:tasks.rados.rados.0.smithi033.stdout:1274: writing smithi03385152-5 from 3563520 to 4186112 tid 3 2024-09-21T09:55:20.596 INFO:tasks.rados.rados.0.smithi033.stdout:1274: writing smithi03385152-5 from 4186112 to 4554752 tid 4 2024-09-21T09:55:20.596 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 112 2024-09-21T09:55:20.596 INFO:tasks.rados.rados.0.smithi033.stdout:1274: finishing write tid 1 to smithi03385152-5 2024-09-21T09:55:20.597 INFO:tasks.rados.rados.0.smithi033.stdout:1275: seq_num 424 ranges {1794048=704512,2498560=491520,2990080=434176,3424256=679936} 2024-09-21T09:55:20.614 INFO:tasks.rados.rados.0.smithi033.stdout:1275: writing smithi03385152-10 from 1794048 to 2498560 tid 1 2024-09-21T09:55:20.615 INFO:tasks.rados.rados.0.smithi033.stdout:1275: writing smithi03385152-10 from 2498560 to 2990080 tid 2 2024-09-21T09:55:20.616 INFO:tasks.rados.rados.0.smithi033.stdout:1275: writing smithi03385152-10 from 2990080 to 3424256 tid 3 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1275: writing smithi03385152-10 from 3424256 to 4104192 tid 4 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1276: rmattr oid 30 current snap is 112 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1274: finishing write tid 2 to smithi03385152-5 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1274: finishing write tid 3 to smithi03385152-5 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1274: finishing write tid 4 to smithi03385152-5 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1274: finishing write tid 5 to smithi03385152-5 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1274: finishing write tid 6 to smithi03385152-5 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 170 (ObjNum 423 snap 112 seq_num 423) dirty exists 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1274: left oid 5 (ObjNum 423 snap 112 seq_num 423) 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1275: finishing write tid 1 to smithi03385152-10 2024-09-21T09:55:20.622 INFO:tasks.rados.rados.0.smithi033.stdout:1275: finishing write tid 2 to smithi03385152-10 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1275: finishing write tid 3 to smithi03385152-10 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1274: done (2 left) 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1277: copy_from oid 34 from oid 50 current snap is 112 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1278: read oid 28 snap -1 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1278: expect (ObjNum 369 snap 97 seq_num 369) 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1279: rollback oid 48 current snap is 112 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 48 to 110 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1280: read oid 26 snap -1 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1280: expect (ObjNum 405 snap 110 seq_num 405) 2024-09-21T09:55:20.623 INFO:tasks.rados.rados.0.smithi033.stdout:1281: snap_create 2024-09-21T09:55:20.624 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 808 (ObjNum 387 snap 105 seq_num 387) dirty exists 2024-09-21T09:55:20.627 INFO:tasks.rados.rados.0.smithi033.stdout:1275: finishing write tid 4 to smithi03385152-10 2024-09-21T09:55:20.635 INFO:tasks.rados.rados.0.smithi033.stdout:1275: finishing write tid 5 to smithi03385152-10 2024-09-21T09:55:20.635 INFO:tasks.rados.rados.0.smithi033.stdout:1279: finishing rollback tid 1 to smithi03385152-48 2024-09-21T09:55:20.635 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 0 (ObjNum 49 snap 0 seq_num 1953066355) dirty dne 2024-09-21T09:55:20.637 INFO:tasks.rados.rados.0.smithi033.stdout:1275: finishing write tid 6 to smithi03385152-10 2024-09-21T09:55:20.637 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 642 (ObjNum 424 snap 112 seq_num 424) dirty exists 2024-09-21T09:55:20.637 INFO:tasks.rados.rados.0.smithi033.stdout:1275: left oid 10 (ObjNum 424 snap 112 seq_num 424) 2024-09-21T09:55:20.637 INFO:tasks.rados.rados.0.smithi033.stdout:1277: finishing copy_from racing read to smithi03385152-34 2024-09-21T09:55:20.639 INFO:tasks.rados.rados.0.smithi033.stdout:1277: finishing copy_from to smithi03385152-34 2024-09-21T09:55:20.639 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 629 (ObjNum 402 snap 110 seq_num 402) dirty exists 2024-09-21T09:55:20.663 INFO:tasks.rados.rados.0.smithi033.stdout:1280: expect (ObjNum 405 snap 110 seq_num 405) 2024-09-21T09:55:20.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:20 smithi033 ceph-mon[98546]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:55:20.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:20 smithi033 ceph-mon[98546]: Reconfiguring daemon osd.2 on smithi033 2024-09-21T09:55:20.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:20 smithi033 ceph-mon[96568]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:55:20.750 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:20 smithi033 ceph-mon[96568]: Reconfiguring daemon osd.2 on smithi033 2024-09-21T09:55:20.901 INFO:tasks.rados.rados.0.smithi033.stdout:1278: expect (ObjNum 369 snap 97 seq_num 369) 2024-09-21T09:55:21.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:20 smithi105 ceph-mon[86649]: Reconfiguring osd.2 (monmap changed)... 2024-09-21T09:55:21.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:20 smithi105 ceph-mon[86649]: Reconfiguring daemon osd.2 on smithi033 2024-09-21T09:55:21.577 INFO:tasks.rados.rados.0.smithi033.stdout:1275: done (6 left) 2024-09-21T09:55:21.577 INFO:tasks.rados.rados.0.smithi033.stdout:1276: done (5 left) 2024-09-21T09:55:21.577 INFO:tasks.rados.rados.0.smithi033.stdout:1277: done (4 left) 2024-09-21T09:55:21.577 INFO:tasks.rados.rados.0.smithi033.stdout:1278: done (3 left) 2024-09-21T09:55:21.578 INFO:tasks.rados.rados.0.smithi033.stdout:1279: done (2 left) 2024-09-21T09:55:21.578 INFO:tasks.rados.rados.0.smithi033.stdout:1280: done (1 left) 2024-09-21T09:55:21.578 INFO:tasks.rados.rados.0.smithi033.stdout:1281: done (0 left) 2024-09-21T09:55:21.578 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 113 2024-09-21T09:55:21.578 INFO:tasks.rados.rados.0.smithi033.stdout:1282: seq_num 425 ranges {4276224=720896,4997120=778240,5775360=638976,6414336=188416} 2024-09-21T09:55:21.605 INFO:tasks.rados.rados.0.smithi033.stdout:1282: writing smithi03385152-44 from 4276224 to 4997120 tid 1 2024-09-21T09:55:21.608 INFO:tasks.rados.rados.0.smithi033.stdout:1282: writing smithi03385152-44 from 4997120 to 5775360 tid 2 2024-09-21T09:55:21.612 INFO:tasks.rados.rados.0.smithi033.stdout:1282: writing smithi03385152-44 from 5775360 to 6414336 tid 3 2024-09-21T09:55:21.614 INFO:tasks.rados.rados.0.smithi033.stdout:1282: writing smithi03385152-44 from 6414336 to 6602752 tid 4 2024-09-21T09:55:21.614 INFO:tasks.rados.rados.0.smithi033.stdout:1283: read oid 10 snap -1 2024-09-21T09:55:21.614 INFO:tasks.rados.rados.0.smithi033.stdout:1283: expect (ObjNum 424 snap 112 seq_num 424) 2024-09-21T09:55:21.614 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 113 2024-09-21T09:55:21.614 INFO:tasks.rados.rados.0.smithi033.stdout:1282: finishing write tid 1 to smithi03385152-44 2024-09-21T09:55:21.614 INFO:tasks.rados.rados.0.smithi033.stdout:1282: finishing write tid 2 to smithi03385152-44 2024-09-21T09:55:21.614 INFO:tasks.rados.rados.0.smithi033.stdout:1284: seq_num 426 ranges {6545408=737280,7282688=712704,7995392=802816,8798208=90112} 2024-09-21T09:55:21.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[98546]: pgmap v69: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 7.2 MiB/s wr, 23 op/s 2024-09-21T09:55:21.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[98546]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:55:21.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:21.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:21.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[98546]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:55:21.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:55:21.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:21.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[98546]: Reconfiguring daemon osd.3 on smithi033 2024-09-21T09:55:21.621 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[96568]: pgmap v69: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 7.2 MiB/s wr, 23 op/s 2024-09-21T09:55:21.621 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[96568]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:55:21.621 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:21.621 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:21.621 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[96568]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:55:21.621 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:55:21.621 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:21.621 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-mon[96568]: Reconfiguring daemon osd.3 on smithi033 2024-09-21T09:55:21.653 INFO:tasks.rados.rados.0.smithi033.stdout:1284: writing smithi03385152-28 from 6545408 to 7282688 tid 1 2024-09-21T09:55:21.658 INFO:tasks.rados.rados.0.smithi033.stdout:1284: writing smithi03385152-28 from 7282688 to 7995392 tid 2 2024-09-21T09:55:21.663 INFO:tasks.rados.rados.0.smithi033.stdout:1284: writing smithi03385152-28 from 7995392 to 8798208 tid 3 2024-09-21T09:55:21.663 INFO:tasks.rados.rados.0.smithi033.stdout:1284: writing smithi03385152-28 from 8798208 to 8888320 tid 4 2024-09-21T09:55:21.663 INFO:tasks.rados.rados.0.smithi033.stdout:1285: rollback oid 26 current snap is 113 2024-09-21T09:55:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 26 to 98 2024-09-21T09:55:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:1282: finishing write tid 3 to smithi03385152-44 2024-09-21T09:55:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:1282: finishing write tid 4 to smithi03385152-44 2024-09-21T09:55:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:1282: finishing write tid 5 to smithi03385152-44 2024-09-21T09:55:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:1282: finishing write tid 6 to smithi03385152-44 2024-09-21T09:55:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 643 (ObjNum 425 snap 113 seq_num 425) dirty exists 2024-09-21T09:55:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:1282: left oid 44 (ObjNum 425 snap 113 seq_num 425) 2024-09-21T09:55:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:1283: expect (ObjNum 424 snap 112 seq_num 424) 2024-09-21T09:55:21.806 INFO:tasks.rados.rados.0.smithi033.stdout:1284: finishing write tid 1 to smithi03385152-28 2024-09-21T09:55:21.806 INFO:tasks.rados.rados.0.smithi033.stdout:1284: finishing write tid 2 to smithi03385152-28 2024-09-21T09:55:21.806 INFO:tasks.rados.rados.0.smithi033.stdout:1284: finishing write tid 3 to smithi03385152-28 2024-09-21T09:55:21.806 INFO:tasks.rados.rados.0.smithi033.stdout:1284: finishing write tid 4 to smithi03385152-28 2024-09-21T09:55:21.806 INFO:tasks.rados.rados.0.smithi033.stdout:1284: finishing write tid 5 to smithi03385152-28 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1285: finishing rollback tid 0 to smithi03385152-26 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1285: finishing rollback tid 1 to smithi03385152-26 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1285: finishing rollback tid 2 to smithi03385152-26 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 492 (ObjNum 345 snap 91 seq_num 345) dirty exists 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1284: finishing write tid 6 to smithi03385152-28 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 157 (ObjNum 426 snap 113 seq_num 426) dirty exists 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1284: left oid 28 (ObjNum 426 snap 113 seq_num 426) 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1282: done (3 left) 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1283: done (2 left) 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1284: done (1 left) 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1285: done (0 left) 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1286: rollback oid 15 current snap is 113 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 15 to 111 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 113 2024-09-21T09:55:21.807 INFO:tasks.rados.rados.0.smithi033.stdout:1287: seq_num 427 ranges {0=761856,761856=598016,1359872=606208,1966080=401408} 2024-09-21T09:55:21.811 INFO:tasks.rados.rados.0.smithi033.stdout:1287: writing smithi03385152-37 from 0 to 761856 tid 1 2024-09-21T09:55:21.814 INFO:tasks.rados.rados.0.smithi033.stdout:1287: writing smithi03385152-37 from 761856 to 1359872 tid 2 2024-09-21T09:55:21.817 INFO:tasks.rados.rados.0.smithi033.stdout:1287: writing smithi03385152-37 from 1359872 to 1966080 tid 3 2024-09-21T09:55:21.820 INFO:tasks.rados.rados.0.smithi033.stdout:1287: writing smithi03385152-37 from 1966080 to 2367488 tid 4 2024-09-21T09:55:21.820 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 113 2024-09-21T09:55:21.820 INFO:tasks.rados.rados.0.smithi033.stdout:1286: finishing rollback tid 0 to smithi03385152-15 2024-09-21T09:55:21.820 INFO:tasks.rados.rados.0.smithi033.stdout:1288: seq_num 428 ranges {4235264=778240,5013504=483328,5496832=409600,5906432=417792,6324224=294912} 2024-09-21T09:55:21.848 INFO:tasks.rados.rados.0.smithi033.stdout:1288: writing smithi03385152-40 from 4235264 to 5013504 tid 1 2024-09-21T09:55:21.850 INFO:tasks.rados.rados.0.smithi033.stdout:1288: writing smithi03385152-40 from 5013504 to 5496832 tid 2 2024-09-21T09:55:21.853 INFO:tasks.rados.rados.0.smithi033.stdout:1288: writing smithi03385152-40 from 5496832 to 5906432 tid 3 2024-09-21T09:55:21.855 INFO:tasks.rados.rados.0.smithi033.stdout:1288: writing smithi03385152-40 from 5906432 to 6324224 tid 4 2024-09-21T09:55:21.857 INFO:tasks.rados.rados.0.smithi033.stdout:1288: writing smithi03385152-40 from 6324224 to 6619136 tid 5 2024-09-21T09:55:21.857 INFO:tasks.rados.rados.0.smithi033.stdout:1289: copy_from oid 13 from oid 24 current snap is 113 2024-09-21T09:55:21.857 INFO:tasks.rados.rados.0.smithi033.stdout:1286: finishing rollback tid 1 to smithi03385152-15 2024-09-21T09:55:21.857 INFO:tasks.rados.rados.0.smithi033.stdout:1286: finishing rollback tid 2 to smithi03385152-15 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 639 (ObjNum 361 snap 96 seq_num 361) dirty exists 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1287: finishing write tid 1 to smithi03385152-37 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1287: finishing write tid 2 to smithi03385152-37 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1287: finishing write tid 3 to smithi03385152-37 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1287: finishing write tid 4 to smithi03385152-37 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1287: finishing write tid 5 to smithi03385152-37 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1287: finishing write tid 6 to smithi03385152-37 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 703 (ObjNum 427 snap 113 seq_num 427) dirty exists 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1287: left oid 37 (ObjNum 427 snap 113 seq_num 427) 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1288: finishing write tid 1 to smithi03385152-40 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1288: finishing write tid 2 to smithi03385152-40 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1288: finishing write tid 3 to smithi03385152-40 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1288: finishing write tid 4 to smithi03385152-40 2024-09-21T09:55:21.858 INFO:tasks.rados.rados.0.smithi033.stdout:1286: done (3 left) 2024-09-21T09:55:21.859 INFO:tasks.rados.rados.0.smithi033.stdout:1287: done (2 left) 2024-09-21T09:55:21.859 INFO:tasks.rados.rados.0.smithi033.stdout:1290: read oid 5 snap 109 2024-09-21T09:55:21.859 INFO:tasks.rados.rados.0.smithi033.stdout:1290: expect (ObjNum 90888528 snap 21955 seq_num 89937840) 2024-09-21T09:55:21.859 INFO:tasks.rados.rados.0.smithi033.stdout:1291: snap_remove snap 111 2024-09-21T09:55:21.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:55:21] "GET /metrics HTTP/1.1" 200 36196 "" "Prometheus/2.51.0" 2024-09-21T09:55:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:21 smithi105 ceph-mon[86649]: pgmap v69: 121 pgs: 121 active+clean; 228 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 7.2 MiB/s wr, 23 op/s 2024-09-21T09:55:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:21 smithi105 ceph-mon[86649]: osdmap e294: 8 total, 8 up, 8 in 2024-09-21T09:55:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:21 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:21 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:21 smithi105 ceph-mon[86649]: Reconfiguring osd.3 (monmap changed)... 2024-09-21T09:55:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:21 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:55:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:21 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:22.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:21 smithi105 ceph-mon[86649]: Reconfiguring daemon osd.3 on smithi033 2024-09-21T09:55:22.583 INFO:tasks.rados.rados.0.smithi033.stdout:1291: done (3 left) 2024-09-21T09:55:22.583 INFO:tasks.rados.rados.0.smithi033.stdout:1292: setattr oid 38 current snap is 113 2024-09-21T09:55:22.583 INFO:tasks.rados.rados.0.smithi033.stdout:1290: expect (ObjNum 90888528 snap 21955 seq_num 89937840) 2024-09-21T09:55:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:1288: finishing write tid 5 to smithi03385152-40 2024-09-21T09:55:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:1288: finishing write tid 6 to smithi03385152-40 2024-09-21T09:55:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:1288: finishing write tid 7 to smithi03385152-40 2024-09-21T09:55:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 683 (ObjNum 428 snap 113 seq_num 428) dirty exists 2024-09-21T09:55:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:1288: left oid 40 (ObjNum 428 snap 113 seq_num 428) 2024-09-21T09:55:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:1289: finishing copy_from racing read to smithi03385152-13 2024-09-21T09:55:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:1289: finishing copy_from to smithi03385152-13 2024-09-21T09:55:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 637 (ObjNum 414 snap 110 seq_num 414) dirty exists 2024-09-21T09:55:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:1288: done (3 left) 2024-09-21T09:55:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:1289: done (2 left) 2024-09-21T09:55:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:1290: done (1 left) 2024-09-21T09:55:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 113 2024-09-21T09:55:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:1293: seq_num 429 ranges {0=794624,794624=770048,1564672=622592,2187264=212992} 2024-09-21T09:55:22.597 INFO:tasks.rados.rados.0.smithi033.stdout:1293: writing smithi03385152-3 from 0 to 794624 tid 1 2024-09-21T09:55:22.600 INFO:tasks.rados.rados.0.smithi033.stdout:1293: writing smithi03385152-3 from 794624 to 1564672 tid 2 2024-09-21T09:55:22.604 INFO:tasks.rados.rados.0.smithi033.stdout:1293: writing smithi03385152-3 from 1564672 to 2187264 tid 3 2024-09-21T09:55:22.605 INFO:tasks.rados.rados.0.smithi033.stdout:1293: writing smithi03385152-3 from 2187264 to 2400256 tid 4 2024-09-21T09:55:22.605 INFO:tasks.rados.rados.0.smithi033.stdout:1294: snap_remove snap 101 2024-09-21T09:55:22.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[98546]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:55:22.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:22.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:22.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:55:22.779 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:22.779 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[96568]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:55:22.779 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:22.780 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:22.780 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:55:22.780 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:22 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:22 smithi105 ceph-mon[86649]: osdmap e295: 8 total, 8 up, 8 in 2024-09-21T09:55:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:22 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:22 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:22 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:55:23.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:22 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:23.440 INFO:tasks.rados.rados.0.smithi033.stdout:1294: done (2 left) 2024-09-21T09:55:23.440 INFO:tasks.rados.rados.0.smithi033.stdout:1295: read oid 31 snap -1 2024-09-21T09:55:23.440 INFO:tasks.rados.rados.0.smithi033.stdout:1295: expect (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:55:23.440 INFO:tasks.rados.rados.0.smithi033.stdout:1296: snap_create 2024-09-21T09:55:23.440 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 684 (ObjNum 97 snap 0 seq_num 91071792) dirty exists 2024-09-21T09:55:23.440 INFO:tasks.rados.rados.0.smithi033.stdout:1293: finishing write tid 1 to smithi03385152-3 2024-09-21T09:55:23.441 INFO:tasks.rados.rados.0.smithi033.stdout:1293: finishing write tid 2 to smithi03385152-3 2024-09-21T09:55:23.441 INFO:tasks.rados.rados.0.smithi033.stdout:1293: finishing write tid 3 to smithi03385152-3 2024-09-21T09:55:23.441 INFO:tasks.rados.rados.0.smithi033.stdout:1293: finishing write tid 4 to smithi03385152-3 2024-09-21T09:55:23.441 INFO:tasks.rados.rados.0.smithi033.stdout:1293: finishing write tid 5 to smithi03385152-3 2024-09-21T09:55:23.441 INFO:tasks.rados.rados.0.smithi033.stdout:1293: finishing write tid 6 to smithi03385152-3 2024-09-21T09:55:23.441 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 730 (ObjNum 429 snap 113 seq_num 429) dirty exists 2024-09-21T09:55:23.441 INFO:tasks.rados.rados.0.smithi033.stdout:1293: left oid 3 (ObjNum 429 snap 113 seq_num 429) 2024-09-21T09:55:23.458 INFO:tasks.rados.rados.0.smithi033.stdout:1295: expect (ObjNum 344 snap 91 seq_num 344) 2024-09-21T09:55:23.955 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:23 smithi105 ceph-mon[86649]: Reconfiguring mds.a (monmap changed)... 2024-09-21T09:55:23.955 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:23 smithi105 ceph-mon[86649]: Reconfiguring daemon mds.a on smithi033 2024-09-21T09:55:23.955 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:23 smithi105 ceph-mon[86649]: pgmap v72: 121 pgs: 121 active+clean; 245 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 4.5 MiB/s wr, 23 op/s 2024-09-21T09:55:23.955 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:23 smithi105 ceph-mon[86649]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:55:23.955 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:23 smithi105 ceph-mon[86649]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:55:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[96568]: Reconfiguring mds.a (monmap changed)... 2024-09-21T09:55:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[96568]: Reconfiguring daemon mds.a on smithi033 2024-09-21T09:55:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[96568]: pgmap v72: 121 pgs: 121 active+clean; 245 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 4.5 MiB/s wr, 23 op/s 2024-09-21T09:55:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[96568]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:55:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[96568]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:55:23.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[98546]: Reconfiguring mds.a (monmap changed)... 2024-09-21T09:55:23.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[98546]: Reconfiguring daemon mds.a on smithi033 2024-09-21T09:55:23.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[98546]: pgmap v72: 121 pgs: 121 active+clean; 245 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 4.5 MiB/s wr, 23 op/s 2024-09-21T09:55:23.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[98546]: osdmap e296: 8 total, 8 up, 8 in 2024-09-21T09:55:23.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:23 smithi033 ceph-mon[98546]: osdmap e297: 8 total, 8 up, 8 in 2024-09-21T09:55:24.215 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:1292: done (3 left) 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:1293: done (2 left) 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:1295: done (1 left) 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:1296: done (0 left) 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:1297: rollback oid 42 current snap is 114 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 42 to 110 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:1298: read oid 22 snap -1 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:1298: expect (ObjNum 419 snap 111 seq_num 419) 2024-09-21T09:55:24.444 INFO:tasks.rados.rados.0.smithi033.stdout:1299: snap_remove snap 109 2024-09-21T09:55:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[98546]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:55:24.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:24.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:24.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:24.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[98546]: Reconfiguring daemon mon.b on smithi105 2024-09-21T09:55:24.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[98546]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:55:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[96568]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:55:24.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:24.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:24.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:24.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[96568]: Reconfiguring daemon mon.b on smithi105 2024-09-21T09:55:24.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:24 smithi033 ceph-mon[96568]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:55:25.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:25.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:25.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-mon[86649]: Reconfiguring mon.b (monmap changed)... 2024-09-21T09:55:25.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-21T09:55:25.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-21T09:55:25.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:25.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-mon[86649]: Reconfiguring daemon mon.b on smithi105 2024-09-21T09:55:25.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:24 smithi105 ceph-mon[86649]: osdmap e298: 8 total, 8 up, 8 in 2024-09-21T09:55:25.444 INFO:tasks.rados.rados.0.smithi033.stdout:1299: done (2 left) 2024-09-21T09:55:25.444 INFO:tasks.rados.rados.0.smithi033.stdout:1300: setattr oid 33 current snap is 114 2024-09-21T09:55:25.444 INFO:tasks.rados.rados.0.smithi033.stdout:1297: finishing rollback tid 0 to smithi03385152-42 2024-09-21T09:55:25.444 INFO:tasks.rados.rados.0.smithi033.stdout:1297: finishing rollback tid 1 to smithi03385152-42 2024-09-21T09:55:25.444 INFO:tasks.rados.rados.0.smithi033.stdout:1297: finishing rollback tid 2 to smithi03385152-42 2024-09-21T09:55:25.444 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 687 (ObjNum 393 snap 107 seq_num 393) dirty exists 2024-09-21T09:55:25.444 INFO:tasks.rados.rados.0.smithi033.stdout:1298: expect (ObjNum 419 snap 111 seq_num 419) 2024-09-21T09:55:25.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-mon[86649]: pgmap v75: 121 pgs: 121 active+clean; 245 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 28 MiB/s rd, 5.3 MiB/s wr, 28 op/s 2024-09-21T09:55:25.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:25.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:25.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:55:25.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:55:25.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:25.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-mon[86649]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:55:25.874 INFO:tasks.rados.rados.0.smithi033.stdout:1297: done (2 left) 2024-09-21T09:55:25.874 INFO:tasks.rados.rados.0.smithi033.stdout:1298: done (1 left) 2024-09-21T09:55:25.875 INFO:tasks.rados.rados.0.smithi033.stdout:1301: copy_from oid 12 from oid 18 current snap is 114 2024-09-21T09:55:25.875 INFO:tasks.rados.rados.0.smithi033.stdout:1302: read oid 24 snap -1 2024-09-21T09:55:25.875 INFO:tasks.rados.rados.0.smithi033.stdout:1302: expect (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:25.875 INFO:tasks.rados.rados.0.smithi033.stdout:1303: delete oid 7 current snap is 114 2024-09-21T09:55:25.876 INFO:tasks.rados.rados.0.smithi033.stdout:1301: finishing copy_from to smithi03385152-12 2024-09-21T09:55:25.876 INFO:tasks.rados.rados.0.smithi033.stdout:1301: got expected ENOENT (src dne) 2024-09-21T09:55:25.876 INFO:tasks.rados.rados.0.smithi033.stdout:1301: finishing copy_from racing read to smithi03385152-12 2024-09-21T09:55:25.877 INFO:tasks.rados.rados.0.smithi033.stdout:1301: done (3 left) 2024-09-21T09:55:25.877 INFO:tasks.rados.rados.0.smithi033.stdout:1303: done (2 left) 2024-09-21T09:55:25.877 INFO:tasks.rados.rados.0.smithi033.stdout:1304: snap_remove snap 113 2024-09-21T09:55:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[98546]: pgmap v75: 121 pgs: 121 active+clean; 245 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 28 MiB/s rd, 5.3 MiB/s wr, 28 op/s 2024-09-21T09:55:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:55:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:55:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[98546]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:55:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[96568]: pgmap v75: 121 pgs: 121 active+clean; 245 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 28 MiB/s rd, 5.3 MiB/s wr, 28 op/s 2024-09-21T09:55:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-21T09:55:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-21T09:55:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:25 smithi033 ceph-mon[96568]: osdmap e299: 8 total, 8 up, 8 in 2024-09-21T09:55:26.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:26.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:26.632 INFO:tasks.rados.rados.0.smithi033.stdout:1304: done (2 left) 2024-09-21T09:55:26.632 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 114 2024-09-21T09:55:26.632 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 449 (ObjNum 409 snap 110 seq_num 409) dirty exists 2024-09-21T09:55:26.632 INFO:tasks.rados.rados.0.smithi033.stdout:1305: seq_num 431 ranges {12353536=24576} 2024-09-21T09:55:26.700 INFO:tasks.rados.rados.0.smithi033.stdout:1305: writing smithi03385152-14 from 12353536 to 12378112 tid 1 2024-09-21T09:55:26.700 INFO:tasks.rados.rados.0.smithi033.stdout:1300: done (2 left) 2024-09-21T09:55:26.700 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 114 2024-09-21T09:55:26.700 INFO:tasks.rados.rados.0.smithi033.stdout:1306: seq_num 432 ranges {5906432=40960} 2024-09-21T09:55:26.732 INFO:tasks.rados.rados.0.smithi033.stdout:1306: writing smithi03385152-23 from 5906432 to 5947392 tid 1 2024-09-21T09:55:26.733 INFO:tasks.rados.rados.0.smithi033.stdout:1307: rollback oid 29 current snap is 114 2024-09-21T09:55:26.733 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 29 to 112 2024-09-21T09:55:26.733 INFO:tasks.rados.rados.0.smithi033.stdout:1302: expect (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:26.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:26 smithi105 ceph-mon[86649]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:55:26.951 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:26 smithi105 ceph-mon[86649]: Reconfiguring daemon mgr.x on smithi105 2024-09-21T09:55:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:26 smithi033 ceph-mon[98546]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:55:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:26 smithi033 ceph-mon[98546]: Reconfiguring daemon mgr.x on smithi105 2024-09-21T09:55:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:26 smithi033 ceph-mon[96568]: Reconfiguring mgr.x (monmap changed)... 2024-09-21T09:55:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:26 smithi033 ceph-mon[96568]: Reconfiguring daemon mgr.x on smithi105 2024-09-21T09:55:27.208 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:27.439 INFO:tasks.rados.rados.0.smithi033.stdout:1305: finishing write tid 1 to smithi03385152-14 2024-09-21T09:55:27.439 INFO:tasks.rados.rados.0.smithi033.stdout:1305: finishing write tid 2 to smithi03385152-14 2024-09-21T09:55:27.439 INFO:tasks.rados.rados.0.smithi033.stdout:1305: finishing write tid 3 to smithi03385152-14 2024-09-21T09:55:27.439 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 749 (ObjNum 431 snap 114 seq_num 431) dirty exists 2024-09-21T09:55:27.439 INFO:tasks.rados.rados.0.smithi033.stdout:1305: left oid 14 (ObjNum 431 snap 114 seq_num 431) 2024-09-21T09:55:27.439 INFO:tasks.rados.rados.0.smithi033.stdout:1307: finishing rollback tid 1 to smithi03385152-29 2024-09-21T09:55:27.439 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1306: finishing write tid 1 to smithi03385152-23 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1306: finishing write tid 2 to smithi03385152-23 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1306: finishing write tid 3 to smithi03385152-23 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 687 (ObjNum 432 snap 114 seq_num 432) dirty exists 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1306: left oid 23 (ObjNum 432 snap 114 seq_num 432) 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1302: done (3 left) 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1305: done (2 left) 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1306: done (1 left) 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1307: done (0 left) 2024-09-21T09:55:27.440 INFO:tasks.rados.rados.0.smithi033.stdout:1308: setattr oid 47 current snap is 114 2024-09-21T09:55:27.441 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 114 2024-09-21T09:55:27.441 INFO:tasks.rados.rados.0.smithi033.stdout:1309: seq_num 433 ranges {1941504=57344} 2024-09-21T09:55:27.451 INFO:tasks.rados.rados.0.smithi033.stdout:1309: writing smithi03385152-34 from 1941504 to 1998848 tid 1 2024-09-21T09:55:27.452 INFO:tasks.rados.rados.0.smithi033.stdout:1310: copy_from oid 23 from oid 20 current snap is 114 2024-09-21T09:55:27.452 INFO:tasks.rados.rados.0.smithi033.stdout:1311: read oid 27 snap -1 2024-09-21T09:55:27.452 INFO:tasks.rados.rados.0.smithi033.stdout:1311: expect deleted 2024-09-21T09:55:27.452 INFO:tasks.rados.rados.0.smithi033.stdout:1312: delete oid 32 current snap is 114 2024-09-21T09:55:27.452 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 92 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1310: finishing copy_from to smithi03385152-23 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1310: got expected ENOENT (src dne) 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1310: finishing copy_from racing read to smithi03385152-23 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1308: done (4 left) 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1310: done (3 left) 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1311: done (2 left) 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1312: done (1 left) 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1313: copy_from oid 38 from oid 21 current snap is 114 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1314: copy_from oid 40 from oid 17 current snap is 114 2024-09-21T09:55:27.453 INFO:tasks.rados.rados.0.smithi033.stdout:1315: copy_from oid 47 from oid 45 current snap is 114 2024-09-21T09:55:27.454 INFO:tasks.rados.rados.0.smithi033.stdout:1316: read oid 24 snap -1 2024-09-21T09:55:27.454 INFO:tasks.rados.rados.0.smithi033.stdout:1316: expect (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:27.454 INFO:tasks.rados.rados.0.smithi033.stdout:1317: read oid 20 snap -1 2024-09-21T09:55:27.454 INFO:tasks.rados.rados.0.smithi033.stdout:1317: expect deleted 2024-09-21T09:55:27.454 INFO:tasks.rados.rados.0.smithi033.stdout:1318: rmattr oid 2 current snap is 114 2024-09-21T09:55:27.454 INFO:tasks.rados.rados.0.smithi033.stdout:1318: done (6 left) 2024-09-21T09:55:27.454 INFO:tasks.rados.rados.0.smithi033.stdout:1319: snap_create 2024-09-21T09:55:27.454 INFO:tasks.rados.rados.0.smithi033.stdout:1309: finishing write tid 1 to smithi03385152-34 2024-09-21T09:55:27.460 INFO:tasks.rados.rados.0.smithi033.stdout:1309: finishing write tid 2 to smithi03385152-34 2024-09-21T09:55:27.463 INFO:tasks.rados.rados.0.smithi033.stdout:1309: finishing write tid 3 to smithi03385152-34 2024-09-21T09:55:27.463 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 752 (ObjNum 433 snap 114 seq_num 433) dirty exists 2024-09-21T09:55:27.464 INFO:tasks.rados.rados.0.smithi033.stdout:1309: left oid 34 (ObjNum 433 snap 114 seq_num 433) 2024-09-21T09:55:27.477 INFO:tasks.rados.rados.0.smithi033.stdout:1313: finishing copy_from racing read to smithi03385152-38 2024-09-21T09:55:27.492 INFO:tasks.rados.rados.0.smithi033.stdout:1314: finishing copy_from racing read to smithi03385152-40 2024-09-21T09:55:27.492 INFO:tasks.rados.rados.0.smithi033.stdout:1313: finishing copy_from to smithi03385152-38 2024-09-21T09:55:27.492 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 677 (ObjNum 308 snap 81 seq_num 308) dirty exists 2024-09-21T09:55:27.492 INFO:tasks.rados.rados.0.smithi033.stdout:1314: finishing copy_from to smithi03385152-40 2024-09-21T09:55:27.493 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 488 (ObjNum 338 snap 87 seq_num 338) dirty exists 2024-09-21T09:55:27.499 INFO:tasks.rados.rados.0.smithi033.stdout:1316: expect (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[96568]: pgmap v78: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-21T09:55:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[96568]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:55:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[96568]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:55:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:55:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[96568]: Reconfiguring daemon osd.4 on smithi105 2024-09-21T09:55:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[98546]: pgmap v78: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-21T09:55:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[98546]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:55:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[98546]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:55:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:55:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:27 smithi033 ceph-mon[98546]: Reconfiguring daemon osd.4 on smithi105 2024-09-21T09:55:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:27 smithi105 ceph-mon[86649]: pgmap v78: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-21T09:55:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:27 smithi105 ceph-mon[86649]: osdmap e300: 8 total, 8 up, 8 in 2024-09-21T09:55:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:27 smithi105 ceph-mon[86649]: Reconfiguring osd.4 (monmap changed)... 2024-09-21T09:55:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:55:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:27 smithi105 ceph-mon[86649]: Reconfiguring daemon osd.4 on smithi105 2024-09-21T09:55:28.209 INFO:tasks.rados.rados.0.smithi033.stdout:1315: finishing copy_from racing read to smithi03385152-47 2024-09-21T09:55:28.209 INFO:tasks.rados.rados.0.smithi033.stdout:1315: finishing copy_from to smithi03385152-47 2024-09-21T09:55:28.209 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 642 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:55:28.209 INFO:tasks.rados.rados.0.smithi033.stdout:1309: done (6 left) 2024-09-21T09:55:28.209 INFO:tasks.rados.rados.0.smithi033.stdout:1313: done (5 left) 2024-09-21T09:55:28.209 INFO:tasks.rados.rados.0.smithi033.stdout:1314: done (4 left) 2024-09-21T09:55:28.210 INFO:tasks.rados.rados.0.smithi033.stdout:1315: done (3 left) 2024-09-21T09:55:28.210 INFO:tasks.rados.rados.0.smithi033.stdout:1316: done (2 left) 2024-09-21T09:55:28.210 INFO:tasks.rados.rados.0.smithi033.stdout:1317: done (1 left) 2024-09-21T09:55:28.210 INFO:tasks.rados.rados.0.smithi033.stdout:1319: done (0 left) 2024-09-21T09:55:28.210 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 115 2024-09-21T09:55:28.210 INFO:tasks.rados.rados.0.smithi033.stdout:1320: seq_num 435 ranges {0=90112} 2024-09-21T09:55:28.210 INFO:tasks.rados.rados.0.smithi033.stdout:1320: writing smithi03385152-35 from 0 to 90112 tid 1 2024-09-21T09:55:28.210 INFO:tasks.rados.rados.0.smithi033.stdout:1321: delete oid 36 current snap is 115 2024-09-21T09:55:28.215 INFO:tasks.rados.rados.0.smithi033.stdout:1320: finishing write tid 1 to smithi03385152-35 2024-09-21T09:55:28.215 INFO:tasks.rados.rados.0.smithi033.stdout:1320: finishing write tid 2 to smithi03385152-35 2024-09-21T09:55:28.216 INFO:tasks.rados.rados.0.smithi033.stdout:1320: finishing write tid 3 to smithi03385152-35 2024-09-21T09:55:28.216 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 834 (ObjNum 435 snap 115 seq_num 435) dirty exists 2024-09-21T09:55:28.216 INFO:tasks.rados.rados.0.smithi033.stdout:1320: left oid 35 (ObjNum 435 snap 115 seq_num 435) 2024-09-21T09:55:28.219 INFO:tasks.rados.rados.0.smithi033.stdout:1320: done (1 left) 2024-09-21T09:55:28.219 INFO:tasks.rados.rados.0.smithi033.stdout:1321: done (0 left) 2024-09-21T09:55:28.219 INFO:tasks.rados.rados.0.smithi033.stdout:1322: read oid 47 snap -1 2024-09-21T09:55:28.219 INFO:tasks.rados.rados.0.smithi033.stdout:1322: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:55:28.220 INFO:tasks.rados.rados.0.smithi033.stdout:1323: delete oid 3 current snap is 115 2024-09-21T09:55:28.222 INFO:tasks.rados.rados.0.smithi033.stdout:1323: done (1 left) 2024-09-21T09:55:28.222 INFO:tasks.rados.rados.0.smithi033.stdout:1324: snap_remove snap 112 2024-09-21T09:55:28.441 INFO:tasks.rados.rados.0.smithi033.stdout:1324: done (1 left) 2024-09-21T09:55:28.441 INFO:tasks.rados.rados.0.smithi033.stdout:1325: read oid 33 snap -1 2024-09-21T09:55:28.441 INFO:tasks.rados.rados.0.smithi033.stdout:1325: expect (ObjNum 409 snap 110 seq_num 409) 2024-09-21T09:55:28.441 INFO:tasks.rados.rados.0.smithi033.stdout:1326: snap_create 2024-09-21T09:55:28.441 INFO:tasks.rados.rados.0.smithi033.stdout:1322: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:55:28.715 INFO:tasks.rados.rados.0.smithi033.stdout:1325: expect (ObjNum 409 snap 110 seq_num 409) 2024-09-21T09:55:28.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:28 smithi105 ceph-mon[86649]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:55:28.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:28 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:28.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:28 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:28.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:28 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:55:28.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:28 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:28.916 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:28 smithi105 ceph-mon[86649]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:55:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[98546]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:55:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:55:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[98546]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:55:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[96568]: osdmap e301: 8 total, 8 up, 8 in 2024-09-21T09:55:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:55:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:28 smithi033 ceph-mon[96568]: osdmap e302: 8 total, 8 up, 8 in 2024-09-21T09:55:29.442 INFO:tasks.rados.rados.0.smithi033.stdout:1322: done (2 left) 2024-09-21T09:55:29.442 INFO:tasks.rados.rados.0.smithi033.stdout:1325: done (1 left) 2024-09-21T09:55:29.442 INFO:tasks.rados.rados.0.smithi033.stdout:1326: done (0 left) 2024-09-21T09:55:29.442 INFO:tasks.rados.rados.0.smithi033.stdout:1327: snap_create 2024-09-21T09:55:29.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:29 smithi105 ceph-mon[86649]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:55:29.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:29 smithi105 ceph-mon[86649]: Reconfiguring daemon osd.5 on smithi105 2024-09-21T09:55:29.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:29 smithi105 ceph-mon[86649]: pgmap v81: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-21T09:55:29.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:29 smithi105 ceph-mon[86649]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:55:29.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:29 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:29.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:29 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:29.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:29 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:55:29.980 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:29 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[96568]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:55:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[96568]: Reconfiguring daemon osd.5 on smithi105 2024-09-21T09:55:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[96568]: pgmap v81: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-21T09:55:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[96568]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:55:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:55:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[98546]: Reconfiguring osd.5 (monmap changed)... 2024-09-21T09:55:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[98546]: Reconfiguring daemon osd.5 on smithi105 2024-09-21T09:55:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[98546]: pgmap v81: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 24 op/s 2024-09-21T09:55:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[98546]: osdmap e303: 8 total, 8 up, 8 in 2024-09-21T09:55:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:55:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:29 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:30.250 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:30.443 INFO:tasks.rados.rados.0.smithi033.stdout:1327: done (0 left) 2024-09-21T09:55:30.443 INFO:tasks.rados.rados.0.smithi033.stdout:1328: setattr oid 9 current snap is 117 2024-09-21T09:55:30.444 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 117 2024-09-21T09:55:30.444 INFO:tasks.rados.rados.0.smithi033.stdout:1329: seq_num 438 ranges {5521408=139264} 2024-09-21T09:55:30.476 INFO:tasks.rados.rados.0.smithi033.stdout:1329: writing smithi03385152-40 from 5521408 to 5660672 tid 1 2024-09-21T09:55:30.476 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 117 2024-09-21T09:55:30.476 INFO:tasks.rados.rados.0.smithi033.stdout:1330: seq_num 439 ranges {5947392=155648} 2024-09-21T09:55:30.509 INFO:tasks.rados.rados.0.smithi033.stdout:1330: writing smithi03385152-23 from 5947392 to 6103040 tid 1 2024-09-21T09:55:30.509 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 117 2024-09-21T09:55:30.509 INFO:tasks.rados.rados.0.smithi033.stdout:1331: seq_num 440 ranges {4546560=172032} 2024-09-21T09:55:30.535 INFO:tasks.rados.rados.0.smithi033.stdout:1331: writing smithi03385152-38 from 4546560 to 4718592 tid 1 2024-09-21T09:55:30.535 INFO:tasks.rados.rados.0.smithi033.stdout:1332: read oid 17 snap 98 2024-09-21T09:55:30.535 INFO:tasks.rados.rados.0.smithi033.stdout:1332: expect (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:55:30.535 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 117 2024-09-21T09:55:30.536 INFO:tasks.rados.rados.0.smithi033.stdout:1333: seq_num 441 ranges {2580480=188416} 2024-09-21T09:55:30.551 INFO:tasks.rados.rados.0.smithi033.stdout:1333: writing smithi03385152-4 from 2580480 to 2768896 tid 1 2024-09-21T09:55:30.551 INFO:tasks.rados.rados.0.smithi033.stdout:1334: setattr oid 33 current snap is 117 2024-09-21T09:55:30.551 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 688 (ObjNum 897 snap 0 seq_num 1163894864) dirty exists 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1329: finishing write tid 1 to smithi03385152-40 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1329: finishing write tid 2 to smithi03385152-40 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1329: finishing write tid 3 to smithi03385152-40 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 690 (ObjNum 438 snap 117 seq_num 438) dirty exists 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1329: left oid 40 (ObjNum 438 snap 117 seq_num 438) 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1330: finishing write tid 1 to smithi03385152-23 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1330: finishing write tid 2 to smithi03385152-23 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1330: finishing write tid 3 to smithi03385152-23 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 689 (ObjNum 439 snap 117 seq_num 439) dirty exists 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1330: left oid 23 (ObjNum 439 snap 117 seq_num 439) 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1331: finishing write tid 1 to smithi03385152-38 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1331: finishing write tid 2 to smithi03385152-38 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:1331: finishing write tid 3 to smithi03385152-38 2024-09-21T09:55:30.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 692 (ObjNum 440 snap 117 seq_num 440) dirty exists 2024-09-21T09:55:30.553 INFO:tasks.rados.rados.0.smithi033.stdout:1331: left oid 38 (ObjNum 440 snap 117 seq_num 440) 2024-09-21T09:55:30.553 INFO:tasks.rados.rados.0.smithi033.stdout:1328: done (6 left) 2024-09-21T09:55:30.553 INFO:tasks.rados.rados.0.smithi033.stdout:1329: done (5 left) 2024-09-21T09:55:30.553 INFO:tasks.rados.rados.0.smithi033.stdout:1330: done (4 left) 2024-09-21T09:55:30.553 INFO:tasks.rados.rados.0.smithi033.stdout:1331: done (3 left) 2024-09-21T09:55:30.554 INFO:tasks.rados.rados.0.smithi033.stdout:1335: read oid 13 snap -1 2024-09-21T09:55:30.554 INFO:tasks.rados.rados.0.smithi033.stdout:1335: expect (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:30.554 INFO:tasks.rados.rados.0.smithi033.stdout:1336: read oid 35 snap -1 2024-09-21T09:55:30.554 INFO:tasks.rados.rados.0.smithi033.stdout:1336: expect (ObjNum 435 snap 115 seq_num 435) 2024-09-21T09:55:30.554 INFO:tasks.rados.rados.0.smithi033.stdout:1337: snap_remove snap 116 2024-09-21T09:55:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:30 smithi033 ceph-mon[98546]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:55:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:30 smithi033 ceph-mon[98546]: Reconfiguring daemon osd.6 on smithi105 2024-09-21T09:55:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:30 smithi033 ceph-mon[98546]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:55:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:30 smithi033 ceph-mon[96568]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:55:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:30 smithi033 ceph-mon[96568]: Reconfiguring daemon osd.6 on smithi105 2024-09-21T09:55:30.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:30 smithi033 ceph-mon[96568]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:55:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:30 smithi105 ceph-mon[86649]: Reconfiguring osd.6 (monmap changed)... 2024-09-21T09:55:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:30 smithi105 ceph-mon[86649]: Reconfiguring daemon osd.6 on smithi105 2024-09-21T09:55:30.997 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:30 smithi105 ceph-mon[86649]: osdmap e304: 8 total, 8 up, 8 in 2024-09-21T09:55:31.446 INFO:tasks.rados.rados.0.smithi033.stdout:1337: done (5 left) 2024-09-21T09:55:31.446 INFO:tasks.rados.rados.0.smithi033.stdout:1338: rollback oid 30 current snap is 117 2024-09-21T09:55:31.446 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 30 to 106 2024-09-21T09:55:31.446 INFO:tasks.rados.rados.0.smithi033.stdout:1333: finishing write tid 1 to smithi03385152-4 2024-09-21T09:55:31.446 INFO:tasks.rados.rados.0.smithi033.stdout:1333: finishing write tid 2 to smithi03385152-4 2024-09-21T09:55:31.446 INFO:tasks.rados.rados.0.smithi033.stdout:1332: expect (ObjNum 338 snap 87 seq_num 338) 2024-09-21T09:55:31.769 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 452 (ObjNum 409 snap 110 seq_num 409) dirty exists 2024-09-21T09:55:31.769 INFO:tasks.rados.rados.0.smithi033.stdout:1333: finishing write tid 3 to smithi03385152-4 2024-09-21T09:55:31.769 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 645 (ObjNum 441 snap 117 seq_num 441) dirty exists 2024-09-21T09:55:31.769 INFO:tasks.rados.rados.0.smithi033.stdout:1333: left oid 4 (ObjNum 441 snap 117 seq_num 441) 2024-09-21T09:55:31.769 INFO:tasks.rados.rados.0.smithi033.stdout:1335: expect (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:55:31] "GET /metrics HTTP/1.1" 200 36198 "" "Prometheus/2.51.0" 2024-09-21T09:55:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[98546]: pgmap v84: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:55:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[98546]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[96568]: pgmap v84: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:31 smithi033 ceph-mon[96568]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:55:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:31 smithi105 ceph-mon[86649]: pgmap v84: 121 pgs: 121 active+clean; 260 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-21T09:55:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:32.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:55:32.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:32.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:31 smithi105 ceph-mon[86649]: osdmap e305: 8 total, 8 up, 8 in 2024-09-21T09:55:32.477 INFO:tasks.rados.rados.0.smithi033.stdout:1336: expect (ObjNum 435 snap 115 seq_num 435) 2024-09-21T09:55:32.479 INFO:tasks.rados.rados.0.smithi033.stdout:1338: finishing rollback tid 0 to smithi03385152-30 2024-09-21T09:55:32.479 INFO:tasks.rados.rados.0.smithi033.stdout:1338: finishing rollback tid 1 to smithi03385152-30 2024-09-21T09:55:32.480 INFO:tasks.rados.rados.0.smithi033.stdout:1338: finishing rollback tid 2 to smithi03385152-30 2024-09-21T09:55:32.480 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 842 (ObjNum 387 snap 105 seq_num 387) dirty exists 2024-09-21T09:55:32.480 INFO:tasks.rados.rados.0.smithi033.stdout:1332: done (5 left) 2024-09-21T09:55:32.480 INFO:tasks.rados.rados.0.smithi033.stdout:1333: done (4 left) 2024-09-21T09:55:32.480 INFO:tasks.rados.rados.0.smithi033.stdout:1334: done (3 left) 2024-09-21T09:55:32.480 INFO:tasks.rados.rados.0.smithi033.stdout:1335: done (2 left) 2024-09-21T09:55:32.480 INFO:tasks.rados.rados.0.smithi033.stdout:1336: done (1 left) 2024-09-21T09:55:32.481 INFO:tasks.rados.rados.0.smithi033.stdout:1338: done (0 left) 2024-09-21T09:55:32.481 INFO:tasks.rados.rados.0.smithi033.stdout:1339: copy_from oid 41 from oid 19 current snap is 117 2024-09-21T09:55:32.481 INFO:tasks.rados.rados.0.smithi033.stdout:1340: read oid 24 snap 110 2024-09-21T09:55:32.481 INFO:tasks.rados.rados.0.smithi033.stdout:1340: expect (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:32.481 INFO:tasks.rados.rados.0.smithi033.stdout:1341: snap_create 2024-09-21T09:55:32.513 INFO:tasks.rados.rados.0.smithi033.stdout:1340: expect (ObjNum 414 snap 110 seq_num 414) 2024-09-21T09:55:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[98546]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:55:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[98546]: Reconfiguring daemon osd.7 on smithi105 2024-09-21T09:55:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:55:32.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:32.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[96568]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:55:32.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[96568]: Reconfiguring daemon osd.7 on smithi105 2024-09-21T09:55:32.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:32.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:32.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:55:32.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:32 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:32.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:32 smithi105 ceph-mon[86649]: Reconfiguring osd.7 (monmap changed)... 2024-09-21T09:55:32.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:32 smithi105 ceph-mon[86649]: Reconfiguring daemon osd.7 on smithi105 2024-09-21T09:55:32.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:32 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:32.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:32 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:32.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:32 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:55:32.998 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:32 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:33.223 INFO:tasks.rados.rados.0.smithi033.stdout:1339: finishing copy_from racing read to smithi03385152-41 2024-09-21T09:55:33.224 INFO:tasks.rados.rados.0.smithi033.stdout:1339: finishing copy_from to smithi03385152-41 2024-09-21T09:55:33.224 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 624 (ObjNum 400 snap 108 seq_num 400) dirty exists 2024-09-21T09:55:33.224 INFO:tasks.rados.rados.0.smithi033.stdout:1339: done (2 left) 2024-09-21T09:55:33.224 INFO:tasks.rados.rados.0.smithi033.stdout:1340: done (1 left) 2024-09-21T09:55:33.224 INFO:tasks.rados.rados.0.smithi033.stdout:1341: done (0 left) 2024-09-21T09:55:33.225 INFO:tasks.rados.rados.0.smithi033.stdout:1342: rmattr oid 11 current snap is 118 2024-09-21T09:55:33.225 INFO:tasks.rados.rados.0.smithi033.stdout:1342: done (0 left) 2024-09-21T09:55:33.225 INFO:tasks.rados.rados.0.smithi033.stdout:1343: snap_remove snap 110 2024-09-21T09:55:33.440 INFO:tasks.rados.rados.0.smithi033.stdout:1343: done (0 left) 2024-09-21T09:55:33.440 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 118 2024-09-21T09:55:33.440 INFO:tasks.rados.rados.0.smithi033.stdout:1344: seq_num 442 ranges {6586368=204800} 2024-09-21T09:55:33.478 INFO:tasks.rados.rados.0.smithi033.stdout:1344: writing smithi03385152-30 from 6586368 to 6791168 tid 1 2024-09-21T09:55:33.479 INFO:tasks.rados.rados.0.smithi033.stdout:1345: read oid 44 snap -1 2024-09-21T09:55:33.479 INFO:tasks.rados.rados.0.smithi033.stdout:1345: expect (ObjNum 425 snap 113 seq_num 425) 2024-09-21T09:55:33.479 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 118 2024-09-21T09:55:33.479 INFO:tasks.rados.rados.0.smithi033.stdout:1346: seq_num 443 ranges {0=221184} 2024-09-21T09:55:33.480 INFO:tasks.rados.rados.0.smithi033.stdout:1346: writing smithi03385152-9 from 0 to 221184 tid 1 2024-09-21T09:55:33.480 INFO:tasks.rados.rados.0.smithi033.stdout:1347: read oid 42 snap -1 2024-09-21T09:55:33.480 INFO:tasks.rados.rados.0.smithi033.stdout:1347: expect (ObjNum 393 snap 107 seq_num 393) 2024-09-21T09:55:33.480 INFO:tasks.rados.rados.0.smithi033.stdout:1348: snap_create 2024-09-21T09:55:33.481 INFO:tasks.rados.rados.0.smithi033.stdout:1344: finishing write tid 1 to smithi03385152-30 2024-09-21T09:55:33.481 INFO:tasks.rados.rados.0.smithi033.stdout:1344: finishing write tid 2 to smithi03385152-30 2024-09-21T09:55:33.482 INFO:tasks.rados.rados.0.smithi033.stdout:1346: finishing write tid 1 to smithi03385152-9 2024-09-21T09:55:33.483 INFO:tasks.rados.rados.0.smithi033.stdout:1346: finishing write tid 2 to smithi03385152-9 2024-09-21T09:55:33.484 INFO:tasks.rados.rados.0.smithi033.stdout:1344: finishing write tid 3 to smithi03385152-30 2024-09-21T09:55:33.484 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 851 (ObjNum 442 snap 118 seq_num 442) dirty exists 2024-09-21T09:55:33.484 INFO:tasks.rados.rados.0.smithi033.stdout:1344: left oid 30 (ObjNum 442 snap 118 seq_num 442) 2024-09-21T09:55:33.487 INFO:tasks.rados.rados.0.smithi033.stdout:1346: finishing write tid 3 to smithi03385152-9 2024-09-21T09:55:33.489 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 694 (ObjNum 443 snap 118 seq_num 443) dirty exists 2024-09-21T09:55:33.489 INFO:tasks.rados.rados.0.smithi033.stdout:1346: left oid 9 (ObjNum 443 snap 118 seq_num 443) 2024-09-21T09:55:33.489 INFO:tasks.rados.rados.0.smithi033.stdout:1347: expect (ObjNum 393 snap 107 seq_num 393) 2024-09-21T09:55:33.537 INFO:tasks.rados.rados.0.smithi033.stdout:1345: expect (ObjNum 425 snap 113 seq_num 425) 2024-09-21T09:55:33.844 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:33 smithi105 ceph-mon[86649]: pgmap v87: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-21T09:55:33.844 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:33 smithi105 ceph-mon[86649]: Reconfiguring mds.b (monmap changed)... 2024-09-21T09:55:33.845 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:33 smithi105 ceph-mon[86649]: Reconfiguring daemon mds.b on smithi105 2024-09-21T09:55:33.845 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:33 smithi105 ceph-mon[86649]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:55:33.845 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:33 smithi105 ceph-mon[86649]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:55:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[98546]: pgmap v87: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-21T09:55:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[98546]: Reconfiguring mds.b (monmap changed)... 2024-09-21T09:55:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[98546]: Reconfiguring daemon mds.b on smithi105 2024-09-21T09:55:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[98546]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:55:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[98546]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:55:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[96568]: pgmap v87: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-21T09:55:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[96568]: Reconfiguring mds.b (monmap changed)... 2024-09-21T09:55:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[96568]: Reconfiguring daemon mds.b on smithi105 2024-09-21T09:55:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[96568]: osdmap e306: 8 total, 8 up, 8 in 2024-09-21T09:55:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:33 smithi033 ceph-mon[96568]: osdmap e307: 8 total, 8 up, 8 in 2024-09-21T09:55:34.443 INFO:tasks.rados.rados.0.smithi033.stdout:1344: done (4 left) 2024-09-21T09:55:34.444 INFO:tasks.rados.rados.0.smithi033.stdout:1345: done (3 left) 2024-09-21T09:55:34.444 INFO:tasks.rados.rados.0.smithi033.stdout:1346: done (2 left) 2024-09-21T09:55:34.444 INFO:tasks.rados.rados.0.smithi033.stdout:1347: done (1 left) 2024-09-21T09:55:34.444 INFO:tasks.rados.rados.0.smithi033.stdout:1348: done (0 left) 2024-09-21T09:55:34.444 INFO:tasks.rados.rados.0.smithi033.stdout:1349: snap_remove snap 98 2024-09-21T09:55:34.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[98546]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:55:35.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:55:35.249 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:34 smithi033 ceph-mon[96568]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:55:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-21T09:55:35.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:34 smithi105 ceph-mon[86649]: osdmap e308: 8 total, 8 up, 8 in 2024-09-21T09:55:35.456 INFO:tasks.rados.rados.0.smithi033.stdout:1349: done (0 left) 2024-09-21T09:55:35.456 INFO:tasks.rados.rados.0.smithi033.stdout:1350: rollback oid 10 current snap is 119 2024-09-21T09:55:35.456 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 106 2024-09-21T09:55:35.456 INFO:tasks.rados.rados.0.smithi033.stdout:1351: snap_remove snap 117 2024-09-21T09:55:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all mon 2024-09-21T09:55:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all crash 2024-09-21T09:55:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:55:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: Upgrade: osd.0 is safe to restart 2024-09-21T09:55:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: pgmap v90: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-21T09:55:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:55:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-mon[86649]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:55:36.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:36.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:35.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all mon 2024-09-21T09:55:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all crash 2024-09-21T09:55:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:55:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: Upgrade: osd.0 is safe to restart 2024-09-21T09:55:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: pgmap v90: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-21T09:55:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[96568]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all mon 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all crash 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: Upgrade: osd.0 is safe to restart 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: pgmap v90: 121 pgs: 121 active+clean; 273 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 3.9 MiB/s wr, 29 op/s 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:36.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:35 smithi033 ceph-mon[98546]: osdmap e309: 8 total, 8 up, 8 in 2024-09-21T09:55:36.446 INFO:tasks.rados.rados.0.smithi033.stdout:1351: done (1 left) 2024-09-21T09:55:36.447 INFO:tasks.rados.rados.0.smithi033.stdout:1352: snap_create 2024-09-21T09:55:36.447 INFO:tasks.rados.rados.0.smithi033.stdout:1350: finishing rollback tid 0 to smithi03385152-10 2024-09-21T09:55:36.447 INFO:tasks.rados.rados.0.smithi033.stdout:1350: finishing rollback tid 1 to smithi03385152-10 2024-09-21T09:55:36.447 INFO:tasks.rados.rados.0.smithi033.stdout:1350: finishing rollback tid 2 to smithi03385152-10 2024-09-21T09:55:36.447 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 645 (ObjNum 250 snap 65 seq_num 250) dirty exists 2024-09-21T09:55:37.198 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:36 smithi033 ceph-mon[96568]: Upgrade: Updating osd.0 2024-09-21T09:55:37.234 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:36 smithi033 ceph-mon[96568]: Deploying daemon osd.0 on smithi033 2024-09-21T09:55:37.235 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:36 smithi033 ceph-mon[96568]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:55:37.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:36 smithi033 ceph-mon[98546]: Upgrade: Updating osd.0 2024-09-21T09:55:37.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:36 smithi033 ceph-mon[98546]: Deploying daemon osd.0 on smithi033 2024-09-21T09:55:37.235 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:36 smithi033 ceph-mon[98546]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:55:37.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:36 smithi105 ceph-mon[86649]: Upgrade: Updating osd.0 2024-09-21T09:55:37.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:36 smithi105 ceph-mon[86649]: Deploying daemon osd.0 on smithi033 2024-09-21T09:55:37.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:36 smithi105 ceph-mon[86649]: osdmap e310: 8 total, 8 up, 8 in 2024-09-21T09:55:37.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:37.451 INFO:tasks.rados.rados.0.smithi033.stdout:1350: done (1 left) 2024-09-21T09:55:37.451 INFO:tasks.rados.rados.0.smithi033.stdout:1352: done (0 left) 2024-09-21T09:55:37.451 INFO:tasks.rados.rados.0.smithi033.stdout:1353: copy_from oid 36 from oid 33 current snap is 120 2024-09-21T09:55:37.451 INFO:tasks.rados.rados.0.smithi033.stdout:1354: rollback oid 44 current snap is 120 2024-09-21T09:55:37.452 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 44 to 115 2024-09-21T09:55:37.452 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 120 2024-09-21T09:55:37.452 INFO:tasks.rados.rados.0.smithi033.stdout:1355: seq_num 444 ranges {5521408=237568} 2024-09-21T09:55:37.484 INFO:tasks.rados.rados.0.smithi033.stdout:1355: writing smithi03385152-8 from 5521408 to 5758976 tid 1 2024-09-21T09:55:37.484 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 120 2024-09-21T09:55:37.484 INFO:tasks.rados.rados.0.smithi033.stdout:1354: finishing rollback tid 0 to smithi03385152-44 2024-09-21T09:55:37.484 INFO:tasks.rados.rados.0.smithi033.stdout:1354: finishing rollback tid 1 to smithi03385152-44 2024-09-21T09:55:37.485 INFO:tasks.rados.rados.0.smithi033.stdout:1354: finishing rollback tid 2 to smithi03385152-44 2024-09-21T09:55:37.485 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 648 (ObjNum 425 snap 113 seq_num 425) dirty exists 2024-09-21T09:55:37.485 INFO:tasks.rados.rados.0.smithi033.stdout:1356: seq_num 445 ranges {221184=253952} 2024-09-21T09:55:37.487 INFO:tasks.rados.rados.0.smithi033.stdout:1356: writing smithi03385152-9 from 221184 to 475136 tid 1 2024-09-21T09:55:37.487 INFO:tasks.rados.rados.0.smithi033.stdout:1354: done (3 left) 2024-09-21T09:55:37.487 INFO:tasks.rados.rados.0.smithi033.stdout:1357: setattr oid 37 current snap is 120 2024-09-21T09:55:37.488 INFO:tasks.rados.rados.0.smithi033.stdout:1358: snap_remove snap 115 2024-09-21T09:55:38.193 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:37 smithi033 ceph-mon[98546]: pgmap v93: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-21T09:55:38.193 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:37 smithi033 ceph-mon[98546]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:55:38.193 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:37 smithi033 ceph-mon[96568]: pgmap v93: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-21T09:55:38.193 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:37 smithi033 ceph-mon[96568]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:55:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:37 smithi105 ceph-mon[86649]: pgmap v93: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-21T09:55:38.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:37 smithi105 ceph-mon[86649]: osdmap e311: 8 total, 8 up, 8 in 2024-09-21T09:55:38.443 INFO:tasks.rados.rados.0.smithi033.stdout:1358: done (4 left) 2024-09-21T09:55:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 120 2024-09-21T09:55:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:1353: finishing copy_from racing read to smithi03385152-36 2024-09-21T09:55:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:1359: seq_num 446 ranges {1794048=278528} 2024-09-21T09:55:38.455 INFO:tasks.rados.rados.0.smithi033.stdout:1359: writing smithi03385152-42 from 1794048 to 2072576 tid 1 2024-09-21T09:55:38.456 INFO:tasks.rados.rados.0.smithi033.stdout:1360: setattr oid 31 current snap is 120 2024-09-21T09:55:38.456 INFO:tasks.rados.rados.0.smithi033.stdout:1356: finishing write tid 1 to smithi03385152-9 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:1355: finishing write tid 1 to smithi03385152-8 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:1355: finishing write tid 2 to smithi03385152-8 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:1356: finishing write tid 2 to smithi03385152-9 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 704 (ObjNum 427 snap 113 seq_num 427) dirty exists 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:1355: finishing write tid 3 to smithi03385152-8 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 641 (ObjNum 444 snap 120 seq_num 444) dirty exists 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:1355: left oid 8 (ObjNum 444 snap 120 seq_num 444) 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:1353: finishing copy_from to smithi03385152-36 2024-09-21T09:55:38.457 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 452 (ObjNum 409 snap 110 seq_num 409) dirty exists 2024-09-21T09:55:38.458 INFO:tasks.rados.rados.0.smithi033.stdout:1356: finishing write tid 3 to smithi03385152-9 2024-09-21T09:55:38.458 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 696 (ObjNum 445 snap 120 seq_num 445) dirty exists 2024-09-21T09:55:38.458 INFO:tasks.rados.rados.0.smithi033.stdout:1356: left oid 9 (ObjNum 445 snap 120 seq_num 445) 2024-09-21T09:55:38.459 INFO:tasks.rados.rados.0.smithi033.stdout:1353: done (5 left) 2024-09-21T09:55:38.459 INFO:tasks.rados.rados.0.smithi033.stdout:1355: done (4 left) 2024-09-21T09:55:38.459 INFO:tasks.rados.rados.0.smithi033.stdout:1356: done (3 left) 2024-09-21T09:55:38.459 INFO:tasks.rados.rados.0.smithi033.stdout:1357: done (2 left) 2024-09-21T09:55:38.459 INFO:tasks.rados.rados.0.smithi033.stdout:1361: snap_remove snap 106 2024-09-21T09:55:38.462 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:38 smithi033 systemd[1]: Stopping Ceph osd.0 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:55:38.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:38 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[43436]: 2024-09-21T09:55:38.459+0000 7fe383d23640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:55:38.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:38 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[43436]: 2024-09-21T09:55:38.459+0000 7fe383d23640 -1 osd.0 312 *** Got signal Terminated *** 2024-09-21T09:55:38.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:38 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[43436]: 2024-09-21T09:55:38.459+0000 7fe383d23640 -1 osd.0 312 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:1359: finishing write tid 1 to smithi03385152-42 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:1359: finishing write tid 2 to smithi03385152-42 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 870 (ObjNum 344 snap 91 seq_num 344) dirty exists 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:1359: finishing write tid 3 to smithi03385152-42 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 698 (ObjNum 446 snap 120 seq_num 446) dirty exists 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:1359: left oid 42 (ObjNum 446 snap 120 seq_num 446) 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:1359: done (2 left) 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:1360: done (1 left) 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:1361: done (0 left) 2024-09-21T09:55:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 120 2024-09-21T09:55:39.454 INFO:tasks.rados.rados.0.smithi033.stdout:1362: seq_num 447 ranges {0=294912} 2024-09-21T09:55:39.455 INFO:tasks.rados.rados.0.smithi033.stdout:1362: writing smithi03385152-29 from 0 to 294912 tid 1 2024-09-21T09:55:39.455 INFO:tasks.rados.rados.0.smithi033.stdout:1363: copy_from oid 14 from oid 8 current snap is 120 2024-09-21T09:55:39.455 INFO:tasks.rados.rados.0.smithi033.stdout:1364: setattr oid 25 current snap is 120 2024-09-21T09:55:39.457 INFO:tasks.rados.rados.0.smithi033.stdout:1365: rollback oid 45 current snap is 120 2024-09-21T09:55:39.457 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 45 to 114 2024-09-21T09:55:39.457 INFO:tasks.rados.rados.0.smithi033.stdout:1366: rollback oid 50 current snap is 120 2024-09-21T09:55:39.457 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 50 to 119 2024-09-21T09:55:39.457 INFO:tasks.rados.rados.0.smithi033.stdout:1367: delete oid 6 current snap is 120 2024-09-21T09:55:39.469 INFO:tasks.rados.rados.0.smithi033.stdout:1367: done (5 left) 2024-09-21T09:55:39.469 INFO:tasks.rados.rados.0.smithi033.stdout:1368: read oid 34 snap -1 2024-09-21T09:55:39.469 INFO:tasks.rados.rados.0.smithi033.stdout:1368: expect (ObjNum 433 snap 114 seq_num 433) 2024-09-21T09:55:39.469 INFO:tasks.rados.rados.0.smithi033.stdout:1369: read oid 27 snap 118 2024-09-21T09:55:39.470 INFO:tasks.rados.rados.0.smithi033.stdout:1369: expect deleted 2024-09-21T09:55:39.470 INFO:tasks.rados.rados.0.smithi033.stdout:1370: read oid 47 snap 114 2024-09-21T09:55:39.470 INFO:tasks.rados.rados.0.smithi033.stdout:1370: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:55:39.470 INFO:tasks.rados.rados.0.smithi033.stdout:1371: copy_from oid 41 from oid 1 current snap is 120 2024-09-21T09:55:39.470 INFO:tasks.rados.rados.0.smithi033.stdout:1372: copy_from oid 44 from oid 40 current snap is 120 2024-09-21T09:55:39.470 INFO:tasks.rados.rados.0.smithi033.stdout:1373: delete oid 19 current snap is 120 2024-09-21T09:55:39.471 INFO:tasks.rados.rados.0.smithi033.stdout:1369: done (10 left) 2024-09-21T09:55:39.471 INFO:tasks.rados.rados.0.smithi033.stdout:1373: done (9 left) 2024-09-21T09:55:39.471 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 120 2024-09-21T09:55:39.471 INFO:tasks.rados.rados.0.smithi033.stdout:1374: seq_num 450 ranges {0=344064} 2024-09-21T09:55:39.473 INFO:tasks.rados.rados.0.smithi033.stdout:1374: writing smithi03385152-3 from 0 to 344064 tid 1 2024-09-21T09:55:39.474 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 120 2024-09-21T09:55:39.474 INFO:tasks.rados.rados.0.smithi033.stdout:1365: finishing rollback tid 0 to smithi03385152-45 2024-09-21T09:55:39.474 INFO:tasks.rados.rados.0.smithi033.stdout:1365: finishing rollback tid 1 to smithi03385152-45 2024-09-21T09:55:39.474 INFO:tasks.rados.rados.0.smithi033.stdout:1365: finishing rollback tid 2 to smithi03385152-45 2024-09-21T09:55:39.474 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 778 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:55:39.474 INFO:tasks.rados.rados.0.smithi033.stdout:1375: seq_num 451 ranges {6791168=360448} 2024-09-21T09:55:39.513 INFO:tasks.rados.rados.0.smithi033.stdout:1375: writing smithi03385152-30 from 6791168 to 7151616 tid 1 2024-09-21T09:55:39.513 INFO:tasks.rados.rados.0.smithi033.stdout:1365: done (10 left) 2024-09-21T09:55:39.513 INFO:tasks.rados.rados.0.smithi033.stdout:1376: copy_from oid 24 from oid 32 current snap is 120 2024-09-21T09:55:39.513 INFO:tasks.rados.rados.0.smithi033.stdout:1368: expect (ObjNum 433 snap 114 seq_num 433) 2024-09-21T09:55:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:1374: finishing write tid 1 to smithi03385152-3 2024-09-21T09:55:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:1363: finishing copy_from racing read to smithi03385152-14 2024-09-21T09:55:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:1374: finishing write tid 2 to smithi03385152-3 2024-09-21T09:55:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:1374: finishing write tid 3 to smithi03385152-3 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 780 (ObjNum 450 snap 120 seq_num 450) dirty exists 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1374: left oid 3 (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1363: finishing copy_from to smithi03385152-14 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 641 (ObjNum 444 snap 120 seq_num 444) dirty exists 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1376: finishing copy_from to smithi03385152-24 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1376: got expected ENOENT (src dne) 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1376: finishing copy_from racing read to smithi03385152-24 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1363: done (10 left) 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1368: done (9 left) 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1374: done (8 left) 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1376: done (7 left) 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1377: read oid 26 snap -1 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1377: expect (ObjNum 345 snap 91 seq_num 345) 2024-09-21T09:55:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:1378: snap_remove snap 118 2024-09-21T09:55:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:39 smithi033 ceph-mon[98546]: pgmap v96: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-21T09:55:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:39 smithi033 ceph-mon[98546]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:55:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:39 smithi033 ceph-mon[98546]: osd.0 marked itself down and dead 2024-09-21T09:55:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:39 smithi033 ceph-mon[96568]: pgmap v96: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-21T09:55:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:39 smithi033 ceph-mon[96568]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:55:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:39 smithi033 ceph-mon[96568]: osd.0 marked itself down and dead 2024-09-21T09:55:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:39 smithi105 ceph-mon[86649]: pgmap v96: 121 pgs: 121 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 1.0 MiB/s wr, 5 op/s 2024-09-21T09:55:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:39 smithi105 ceph-mon[86649]: osdmap e312: 8 total, 8 up, 8 in 2024-09-21T09:55:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:39 smithi105 ceph-mon[86649]: osd.0 marked itself down and dead 2024-09-21T09:55:40.284 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:40 smithi033 podman[103204]: 2024-09-21 09:55:40.00465992 +0000 UTC m=+1.670318431 container died 694de6e7dd0c3bae34a191fb2e48936ce42bc7b3515848bf6863a79d56195126 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T09:55:40.464 INFO:tasks.rados.rados.0.smithi033.stdout:1377: expect (ObjNum 345 snap 91 seq_num 345) 2024-09-21T09:55:40.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:40 smithi105 ceph-mon[86649]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:55:40.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:40 smithi105 ceph-mon[86649]: osdmap e313: 8 total, 7 up, 8 in 2024-09-21T09:55:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:40.599 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 630 (ObjNum 33 snap 0 seq_num 0) dirty exists 2024-09-21T09:55:40.599 INFO:tasks.rados.rados.0.smithi033.stdout:1366: finishing rollback tid 0 to smithi03385152-50 2024-09-21T09:55:40.599 INFO:tasks.rados.rados.0.smithi033.stdout:1366: finishing rollback tid 1 to smithi03385152-50 2024-09-21T09:55:40.599 INFO:tasks.rados.rados.0.smithi033.stdout:1364: done (8 left) 2024-09-21T09:55:40.599 INFO:tasks.rados.rados.0.smithi033.stdout:1377: done (7 left) 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1378: done (6 left) 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1379: setattr oid 42 current snap is 120 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1366: finishing rollback tid 2 to smithi03385152-50 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 633 (ObjNum 402 snap 110 seq_num 402) dirty exists 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1362: finishing write tid 1 to smithi03385152-29 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1362: finishing write tid 2 to smithi03385152-29 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1375: finishing write tid 1 to smithi03385152-30 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1362: finishing write tid 3 to smithi03385152-29 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 872 (ObjNum 447 snap 120 seq_num 447) dirty exists 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1362: left oid 29 (ObjNum 447 snap 120 seq_num 447) 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1375: finishing write tid 2 to smithi03385152-30 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1375: finishing write tid 3 to smithi03385152-30 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 875 (ObjNum 451 snap 120 seq_num 451) dirty exists 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1375: left oid 30 (ObjNum 451 snap 120 seq_num 451) 2024-09-21T09:55:40.600 INFO:tasks.rados.rados.0.smithi033.stdout:1371: finishing copy_from racing read to smithi03385152-41 2024-09-21T09:55:40.601 INFO:tasks.rados.rados.0.smithi033.stdout:1371: finishing copy_from to smithi03385152-41 2024-09-21T09:55:40.601 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 636 (ObjNum 404 snap 110 seq_num 404) dirty exists 2024-09-21T09:55:40.601 INFO:tasks.rados.rados.0.smithi033.stdout:1370: expect (ObjNum 279 snap 69 seq_num 279) 2024-09-21T09:55:40.641 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:40 smithi033 ceph-mon[98546]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:55:40.642 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:40 smithi033 ceph-mon[98546]: osdmap e313: 8 total, 7 up, 8 in 2024-09-21T09:55:40.642 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:40 smithi033 podman[103204]: 2024-09-21 09:55:40.372225696 +0000 UTC m=+2.037884196 container cleanup 694de6e7dd0c3bae34a191fb2e48936ce42bc7b3515848bf6863a79d56195126 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T09:55:40.642 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:40 smithi033 bash[103204]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0 2024-09-21T09:55:40.642 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:40 smithi033 ceph-mon[96568]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:55:40.642 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:40 smithi033 ceph-mon[96568]: osdmap e313: 8 total, 7 up, 8 in 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:1372: finishing copy_from racing read to smithi03385152-44 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:1372: finishing copy_from to smithi03385152-44 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 690 (ObjNum 438 snap 117 seq_num 438) dirty exists 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:1362: done (6 left) 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:1366: done (5 left) 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:1370: done (4 left) 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:1371: done (3 left) 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:1372: done (2 left) 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:1375: done (1 left) 2024-09-21T09:55:40.885 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 120 2024-09-21T09:55:40.886 INFO:tasks.rados.rados.0.smithi033.stdout:1380: seq_num 452 ranges {0=376832} 2024-09-21T09:55:40.887 INFO:tasks.rados.rados.0.smithi033.stdout:1380: writing smithi03385152-6 from 0 to 376832 tid 1 2024-09-21T09:55:40.887 INFO:tasks.rados.rados.0.smithi033.stdout:1381: snap_remove snap 119 2024-09-21T09:55:40.993 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:40 smithi033 podman[103225]: 2024-09-21 09:55:40.640384507 +0000 UTC m=+0.635089955 container remove 694de6e7dd0c3bae34a191fb2e48936ce42bc7b3515848bf6863a79d56195126 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T09:55:41.112 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:55:41.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:40 smithi033 podman[103332]: 2024-09-21 09:55:40.896939373 +0000 UTC m=+0.025641177 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:55:41.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:41 smithi033 podman[103332]: 2024-09-21 09:55:41.02471195 +0000 UTC m=+0.153413743 container create 6ac620ecee5aa1ef46bdd5899db33839bcd507c5b834cc9f3cff68773e2f4823 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T09:55:41.466 INFO:tasks.rados.rados.0.smithi033.stdout:1381: done (2 left) 2024-09-21T09:55:41.467 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 120 2024-09-21T09:55:41.467 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 699 (ObjNum 446 snap 120 seq_num 446) dirty exists 2024-09-21T09:55:41.467 INFO:tasks.rados.rados.0.smithi033.stdout:1382: seq_num 453 ranges {966656=393216} 2024-09-21T09:55:41.474 INFO:tasks.rados.rados.0.smithi033.stdout:1382: writing smithi03385152-46 from 966656 to 1359872 tid 1 2024-09-21T09:55:41.474 INFO:tasks.rados.rados.0.smithi033.stdout:1379: done (2 left) 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1383: rollback oid 39 current snap is 120 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 39 to 114 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1380: finishing write tid 1 to smithi03385152-6 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1380: finishing write tid 2 to smithi03385152-6 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1380: finishing write tid 3 to smithi03385152-6 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 472 (ObjNum 452 snap 120 seq_num 452) dirty exists 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1380: left oid 6 (ObjNum 452 snap 120 seq_num 452) 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1380: done (2 left) 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1384: copy_from oid 28 from oid 31 current snap is 120 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 120 2024-09-21T09:55:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1385: seq_num 454 ranges {294912=409600} 2024-09-21T09:55:41.478 INFO:tasks.rados.rados.0.smithi033.stdout:1385: writing smithi03385152-29 from 294912 to 704512 tid 1 2024-09-21T09:55:41.478 INFO:tasks.rados.rados.0.smithi033.stdout:1386: read oid 3 snap -1 2024-09-21T09:55:41.479 INFO:tasks.rados.rados.0.smithi033.stdout:1386: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:55:41.479 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 120 2024-09-21T09:55:41.479 INFO:tasks.rados.rados.0.smithi033.stdout:1387: seq_num 455 ranges {5758976=425984} 2024-09-21T09:55:41.513 INFO:tasks.rados.rados.0.smithi033.stdout:1387: writing smithi03385152-14 from 5758976 to 6184960 tid 1 2024-09-21T09:55:41.513 INFO:tasks.rados.rados.0.smithi033.stdout:1388: delete oid 34 current snap is 120 2024-09-21T09:55:41.513 INFO:tasks.rados.rados.0.smithi033.stdout:1383: finishing rollback tid 1 to smithi03385152-39 2024-09-21T09:55:41.513 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 0 (ObjNum 113 snap 0 seq_num 91277328) dirty dne 2024-09-21T09:55:41.513 INFO:tasks.rados.rados.0.smithi033.stdout:1382: finishing write tid 1 to smithi03385152-46 2024-09-21T09:55:41.513 INFO:tasks.rados.rados.0.smithi033.stdout:1382: finishing write tid 2 to smithi03385152-46 2024-09-21T09:55:41.513 INFO:tasks.rados.rados.0.smithi033.stdout:1382: finishing write tid 3 to smithi03385152-46 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 803 (ObjNum 453 snap 120 seq_num 453) dirty exists 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:1382: left oid 46 (ObjNum 453 snap 120 seq_num 453) 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:1385: finishing write tid 1 to smithi03385152-29 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:1385: finishing write tid 2 to smithi03385152-29 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:1384: finishing copy_from racing read to smithi03385152-28 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:1384: finishing copy_from to smithi03385152-28 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 870 (ObjNum 344 snap 91 seq_num 344) dirty exists 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:1385: finishing write tid 3 to smithi03385152-29 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 877 (ObjNum 454 snap 120 seq_num 454) dirty exists 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:1385: left oid 29 (ObjNum 454 snap 120 seq_num 454) 2024-09-21T09:55:41.514 INFO:tasks.rados.rados.0.smithi033.stdout:1386: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1387: finishing write tid 1 to smithi03385152-14 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1387: finishing write tid 2 to smithi03385152-14 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1387: finishing write tid 3 to smithi03385152-14 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 805 (ObjNum 455 snap 120 seq_num 455) dirty exists 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1387: left oid 14 (ObjNum 455 snap 120 seq_num 455) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1382: done (6 left) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1383: done (5 left) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1384: done (4 left) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1385: done (3 left) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1386: done (2 left) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1387: done (1 left) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1388: done (0 left) 2024-09-21T09:55:41.529 INFO:tasks.rados.rados.0.smithi033.stdout:1389: snap_create 2024-09-21T09:55:41.620 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:41 smithi033 ceph-mon[96568]: pgmap v99: 121 pgs: 16 stale+active+clean, 105 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:55:41.620 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:41 smithi033 ceph-mon[96568]: osdmap e314: 8 total, 7 up, 8 in 2024-09-21T09:55:41.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:41 smithi033 ceph-mon[98546]: pgmap v99: 121 pgs: 16 stale+active+clean, 105 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:55:41.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:41 smithi033 ceph-mon[98546]: osdmap e314: 8 total, 7 up, 8 in 2024-09-21T09:55:41.620 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:41 smithi033 podman[103332]: 2024-09-21 09:55:41.294004057 +0000 UTC m=+0.422705851 container init 6ac620ecee5aa1ef46bdd5899db33839bcd507c5b834cc9f3cff68773e2f4823 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T09:55:41.620 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:41 smithi033 podman[103332]: 2024-09-21 09:55:41.299245585 +0000 UTC m=+0.427947371 container start 6ac620ecee5aa1ef46bdd5899db33839bcd507c5b834cc9f3cff68773e2f4823 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:55:41.621 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:41 smithi033 podman[103332]: 2024-09-21 09:55:41.410987902 +0000 UTC m=+0.539689692 container attach 6ac620ecee5aa1ef46bdd5899db33839bcd507c5b834cc9f3cff68773e2f4823 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-21T09:55:41.621 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:41 smithi033 podman[103332]: 2024-09-21 09:55:41.501149176 +0000 UTC m=+0.629850971 container died 6ac620ecee5aa1ef46bdd5899db33839bcd507c5b834cc9f3cff68773e2f4823 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-62da54b) 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (2m) 47s ago 6m 15.5M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (2m) 33s ago 6m 82.4M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (8m) 47s ago 8m 15.5M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (8m) 33s ago 8m 19.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (92s) 33s ago 11m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (119s) 47s ago 13m 513M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (78s) 47s ago 13m 40.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (38s) 33s ago 12m 22.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:55:41.727 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (56s) 47s ago 12m 26.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (2m) 47s ago 6m 9663k - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (2m) 33s ago 6m 9915k - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (11m) 47s ago 11m 157M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 694de6e7dd0c 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (11m) 47s ago 11m 167M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (10m) 47s ago 10m 145M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (10m) 47s ago 10m 94.9M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (10m) 33s ago 10m 167M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (9m) 33s ago 9m 193M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (9m) 33s ago 9m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (8m) 33s ago 8m 140M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:55:41.728 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (97s) 33s ago 7m 44.0M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:55:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:41 smithi105 ceph-mon[86649]: pgmap v99: 121 pgs: 16 stale+active+clean, 105 active+clean; 216 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:55:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:41 smithi105 ceph-mon[86649]: osdmap e314: 8 total, 7 up, 8 in 2024-09-21T09:55:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:55:41] "GET /metrics HTTP/1.1" 200 36198 "" "Prometheus/2.51.0" 2024-09-21T09:55:42.082 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:55:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:55:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:55:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:55:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:55:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:55:42.082 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 7 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 9, 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:55:42.083 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:55:42.358 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 podman[103421]: 2024-09-21 09:55:42.12335235 +0000 UTC m=+0.614307677 container remove 6ac620ecee5aa1ef46bdd5899db33839bcd507c5b834cc9f3cff68773e2f4823 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True) 2024-09-21T09:55:42.358 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.0.service: Deactivated successfully. 2024-09-21T09:55:42.358 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 systemd[1]: Stopped Ceph osd.0 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:55:42.358 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.0.service: Consumed 12.387s CPU time. 2024-09-21T09:55:42.402 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:55:42.402 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:55:42.402 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:55:42.402 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:55:42.402 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:55:42.402 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:55:42.403 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:55:42.403 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:55:42.403 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "5/20 daemons upgraded", 2024-09-21T09:55:42.403 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:55:42.403 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:55:42.403 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:55:42.478 INFO:tasks.rados.rados.0.smithi033.stdout:1389: done (0 left) 2024-09-21T09:55:42.478 INFO:tasks.rados.rados.0.smithi033.stdout:1390: rollback oid 30 current snap is 121 2024-09-21T09:55:42.478 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 30 to 120 2024-09-21T09:55:42.478 INFO:tasks.rados.rados.0.smithi033.stdout:1391: snap_remove snap 114 2024-09-21T09:55:42.714 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 systemd[1]: Starting Ceph osd.0 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:55:42.714 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 podman[103578]: 2024-09-21 09:55:42.574586601 +0000 UTC m=+0.115798428 container create 5a4ad256c681417346282de19efe5cc5ecb73a4041bb809b48df02b642c68534 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:55:42.714 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 podman[103578]: 2024-09-21 09:55:42.478028046 +0000 UTC m=+0.019239840 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:55:42.714 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[96568]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.714 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[96568]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.714 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[96568]: osdmap e315: 8 total, 7 up, 8 in 2024-09-21T09:55:42.714 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[96568]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.714 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/217905902' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:42.715 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[98546]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.715 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[98546]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.715 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[98546]: osdmap e315: 8 total, 7 up, 8 in 2024-09-21T09:55:42.715 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[98546]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.715 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/217905902' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:42 smithi105 ceph-mon[86649]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:42 smithi105 ceph-mon[86649]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:42 smithi105 ceph-mon[86649]: osdmap e315: 8 total, 7 up, 8 in 2024-09-21T09:55:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:42 smithi105 ceph-mon[86649]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:42 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/217905902' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:42.977 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 podman[103578]: 2024-09-21 09:55:42.835866179 +0000 UTC m=+0.377077965 container init 5a4ad256c681417346282de19efe5cc5ecb73a4041bb809b48df02b642c68534 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2) 2024-09-21T09:55:42.977 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 podman[103578]: 2024-09-21 09:55:42.841063758 +0000 UTC m=+0.382275542 container start 5a4ad256c681417346282de19efe5cc5ecb73a4041bb809b48df02b642c68534 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:55:42.977 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 podman[103578]: 2024-09-21 09:55:42.960472336 +0000 UTC m=+0.501684124 container attach 5a4ad256c681417346282de19efe5cc5ecb73a4041bb809b48df02b642c68534 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-62da54b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T09:55:43.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:55:43.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 bash[103578]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:55:43.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:55:43.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:42 smithi033 bash[103578]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:55:43.451 INFO:tasks.rados.rados.0.smithi033.stdout:1391: done (1 left) 2024-09-21T09:55:43.451 INFO:tasks.rados.rados.0.smithi033.stdout:1392: delete oid 36 current snap is 121 2024-09-21T09:55:43.451 INFO:tasks.rados.rados.0.smithi033.stdout:1390: finishing rollback tid 0 to smithi03385152-30 2024-09-21T09:55:43.451 INFO:tasks.rados.rados.0.smithi033.stdout:1390: finishing rollback tid 1 to smithi03385152-30 2024-09-21T09:55:43.451 INFO:tasks.rados.rados.0.smithi033.stdout:1390: finishing rollback tid 2 to smithi03385152-30 2024-09-21T09:55:43.451 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 881 (ObjNum 451 snap 120 seq_num 451) dirty exists 2024-09-21T09:55:43.458 INFO:tasks.rados.rados.0.smithi033.stdout:1390: done (1 left) 2024-09-21T09:55:43.458 INFO:tasks.rados.rados.0.smithi033.stdout:1392: done (0 left) 2024-09-21T09:55:43.458 INFO:tasks.rados.rados.0.smithi033.stdout:1393: snap_remove snap 120 2024-09-21T09:55:43.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[98546]: pgmap v102: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.4 MiB/s wr, 32 op/s; 66/312 objects degraded (21.154%) 2024-09-21T09:55:43.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[98546]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:43.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[98546]: Health check failed: Degraded data redundancy: 66/312 objects degraded (21.154%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:43.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[98546]: osdmap e316: 8 total, 7 up, 8 in 2024-09-21T09:55:43.740 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[98546]: osdmap e317: 8 total, 7 up, 8 in 2024-09-21T09:55:43.740 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[96568]: pgmap v102: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.4 MiB/s wr, 32 op/s; 66/312 objects degraded (21.154%) 2024-09-21T09:55:43.740 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[96568]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:43.740 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[96568]: Health check failed: Degraded data redundancy: 66/312 objects degraded (21.154%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:43.741 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[96568]: osdmap e316: 8 total, 7 up, 8 in 2024-09-21T09:55:43.741 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-mon[96568]: osdmap e317: 8 total, 7 up, 8 in 2024-09-21T09:55:43.741 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:55:43.741 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 bash[103578]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:55:43.741 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:55:43.741 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 bash[103578]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:55:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:43 smithi105 ceph-mon[86649]: pgmap v102: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.4 MiB/s wr, 32 op/s; 66/312 objects degraded (21.154%) 2024-09-21T09:55:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:43 smithi105 ceph-mon[86649]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:55:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:43 smithi105 ceph-mon[86649]: Health check failed: Degraded data redundancy: 66/312 objects degraded (21.154%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:43 smithi105 ceph-mon[86649]: osdmap e316: 8 total, 7 up, 8 in 2024-09-21T09:55:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:43 smithi105 ceph-mon[86649]: osdmap e317: 8 total, 7 up, 8 in 2024-09-21T09:55:43.996 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:55:43.996 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 bash[103578]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:55:43.996 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:55:43.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 bash[103578]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:55:43.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T09:55:43.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 bash[103578]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-21T09:55:43.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:55:43.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 bash[103578]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-21T09:55:43.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate[103594]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T09:55:43.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 bash[103578]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-21T09:55:43.997 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:43 smithi033 podman[103578]: 2024-09-21 09:55:43.783924406 +0000 UTC m=+1.325136212 container died 5a4ad256c681417346282de19efe5cc5ecb73a4041bb809b48df02b642c68534 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T09:55:44.449 INFO:tasks.rados.rados.0.smithi033.stdout:1393: done (0 left) 2024-09-21T09:55:44.449 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 121 2024-09-21T09:55:44.449 INFO:tasks.rados.rados.0.smithi033.stdout:1394: seq_num 458 ranges {0=475136} 2024-09-21T09:55:44.452 INFO:tasks.rados.rados.0.smithi033.stdout:1394: writing smithi03385152-32 from 0 to 475136 tid 1 2024-09-21T09:55:44.452 INFO:tasks.rados.rados.0.smithi033.stdout:1395: read oid 23 snap -1 2024-09-21T09:55:44.452 INFO:tasks.rados.rados.0.smithi033.stdout:1395: expect (ObjNum 439 snap 117 seq_num 439) 2024-09-21T09:55:44.452 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 121 2024-09-21T09:55:44.453 INFO:tasks.rados.rados.0.smithi033.stdout:1396: seq_num 459 ranges {1974272=491520} 2024-09-21T09:55:44.467 INFO:tasks.rados.rados.0.smithi033.stdout:1396: writing smithi03385152-41 from 1974272 to 2465792 tid 1 2024-09-21T09:55:44.467 INFO:tasks.rados.rados.0.smithi033.stdout:1397: read oid 36 snap -1 2024-09-21T09:55:44.467 INFO:tasks.rados.rados.0.smithi033.stdout:1397: expect deleted 2024-09-21T09:55:44.467 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 121 2024-09-21T09:55:44.467 INFO:tasks.rados.rados.0.smithi033.stdout:1394: finishing write tid 1 to smithi03385152-32 2024-09-21T09:55:44.467 INFO:tasks.rados.rados.0.smithi033.stdout:1394: finishing write tid 2 to smithi03385152-32 2024-09-21T09:55:44.468 INFO:tasks.rados.rados.0.smithi033.stdout:1394: finishing write tid 3 to smithi03385152-32 2024-09-21T09:55:44.468 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 147 (ObjNum 458 snap 121 seq_num 458) dirty exists 2024-09-21T09:55:44.468 INFO:tasks.rados.rados.0.smithi033.stdout:1394: left oid 32 (ObjNum 458 snap 121 seq_num 458) 2024-09-21T09:55:44.468 INFO:tasks.rados.rados.0.smithi033.stdout:1398: seq_num 460 ranges {4546560=507904} 2024-09-21T09:55:44.495 INFO:tasks.rados.rados.0.smithi033.stdout:1398: writing smithi03385152-21 from 4546560 to 5054464 tid 1 2024-09-21T09:55:44.495 INFO:tasks.rados.rados.0.smithi033.stdout:1394: done (4 left) 2024-09-21T09:55:44.495 INFO:tasks.rados.rados.0.smithi033.stdout:1399: setattr oid 13 current snap is 121 2024-09-21T09:55:44.495 INFO:tasks.rados.rados.0.smithi033.stdout:1396: finishing write tid 1 to smithi03385152-41 2024-09-21T09:55:44.495 INFO:tasks.rados.rados.0.smithi033.stdout:1396: finishing write tid 2 to smithi03385152-41 2024-09-21T09:55:44.495 INFO:tasks.rados.rados.0.smithi033.stdout:1396: finishing write tid 3 to smithi03385152-41 2024-09-21T09:55:44.495 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 701 (ObjNum 459 snap 121 seq_num 459) dirty exists 2024-09-21T09:55:44.495 INFO:tasks.rados.rados.0.smithi033.stdout:1396: left oid 41 (ObjNum 459 snap 121 seq_num 459) 2024-09-21T09:55:44.496 INFO:tasks.rados.rados.0.smithi033.stdout:1395: expect (ObjNum 439 snap 117 seq_num 439) 2024-09-21T09:55:44.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:44.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:44 smithi033 podman[103751]: 2024-09-21 09:55:44.449609111 +0000 UTC m=+0.659469256 container remove 5a4ad256c681417346282de19efe5cc5ecb73a4041bb809b48df02b642c68534 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:55:44.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:44 smithi033 podman[103827]: 2024-09-21 09:55:44.586315924 +0000 UTC m=+0.017330937 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:55:44.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:44 smithi033 podman[103827]: 2024-09-21 09:55:44.698870096 +0000 UTC m=+0.129885095 container create 28dc5f57b5bceaa29c8a5b97d30675e84972fa356f0949756fbb7daa467c94eb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS) 2024-09-21T09:55:45.024 INFO:tasks.rados.rados.0.smithi033.stdout:1398: finishing write tid 1 to smithi03385152-21 2024-09-21T09:55:45.024 INFO:tasks.rados.rados.0.smithi033.stdout:1398: finishing write tid 2 to smithi03385152-21 2024-09-21T09:55:45.024 INFO:tasks.rados.rados.0.smithi033.stdout:1398: finishing write tid 3 to smithi03385152-21 2024-09-21T09:55:45.024 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 691 (ObjNum 460 snap 121 seq_num 460) dirty exists 2024-09-21T09:55:45.024 INFO:tasks.rados.rados.0.smithi033.stdout:1398: left oid 21 (ObjNum 460 snap 121 seq_num 460) 2024-09-21T09:55:45.024 INFO:tasks.rados.rados.0.smithi033.stdout:1395: done (4 left) 2024-09-21T09:55:45.024 INFO:tasks.rados.rados.0.smithi033.stdout:1396: done (3 left) 2024-09-21T09:55:45.025 INFO:tasks.rados.rados.0.smithi033.stdout:1397: done (2 left) 2024-09-21T09:55:45.025 INFO:tasks.rados.rados.0.smithi033.stdout:1398: done (1 left) 2024-09-21T09:55:45.025 INFO:tasks.rados.rados.0.smithi033.stdout:1400: delete oid 39 current snap is 121 2024-09-21T09:55:45.025 INFO:tasks.rados.rados.0.smithi033.stdout:1400: done (1 left) 2024-09-21T09:55:45.025 INFO:tasks.rados.rados.0.smithi033.stdout:1401: rollback oid 41 current snap is 121 2024-09-21T09:55:45.025 INFO:tasks.rados.rados.0.smithi033.stdout:1401: done (1 left) 2024-09-21T09:55:45.025 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 121 2024-09-21T09:55:45.026 INFO:tasks.rados.rados.0.smithi033.stdout:1402: seq_num 462 ranges {7151616=540672} 2024-09-21T09:55:45.070 INFO:tasks.rados.rados.0.smithi033.stdout:1402: writing smithi03385152-30 from 7151616 to 7692288 tid 1 2024-09-21T09:55:45.071 INFO:tasks.rados.rados.0.smithi033.stdout:1403: rmattr oid 10 current snap is 121 2024-09-21T09:55:45.071 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 884 (ObjNum 414 snap 110 seq_num 414) dirty exists 2024-09-21T09:55:45.071 INFO:tasks.rados.rados.0.smithi033.stdout:1399: done (2 left) 2024-09-21T09:55:45.071 INFO:tasks.rados.rados.0.smithi033.stdout:1404: setattr oid 38 current snap is 121 2024-09-21T09:55:45.073 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 647 (ObjNum 250 snap 65 seq_num 250) dirty exists 2024-09-21T09:55:45.073 INFO:tasks.rados.rados.0.smithi033.stdout:1403: done (2 left) 2024-09-21T09:55:45.073 INFO:tasks.rados.rados.0.smithi033.stdout:1405: read oid 14 snap -1 2024-09-21T09:55:45.073 INFO:tasks.rados.rados.0.smithi033.stdout:1405: expect (ObjNum 455 snap 120 seq_num 455) 2024-09-21T09:55:45.073 INFO:tasks.rados.rados.0.smithi033.stdout:1406: snap_create 2024-09-21T09:55:45.074 INFO:tasks.rados.rados.0.smithi033.stdout:1402: finishing write tid 1 to smithi03385152-30 2024-09-21T09:55:45.074 INFO:tasks.rados.rados.0.smithi033.stdout:1402: finishing write tid 2 to smithi03385152-30 2024-09-21T09:55:45.075 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 702 (ObjNum 440 snap 117 seq_num 440) dirty exists 2024-09-21T09:55:45.087 INFO:tasks.rados.rados.0.smithi033.stdout:1402: finishing write tid 3 to smithi03385152-30 2024-09-21T09:55:45.087 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 886 (ObjNum 462 snap 121 seq_num 462) dirty exists 2024-09-21T09:55:45.087 INFO:tasks.rados.rados.0.smithi033.stdout:1402: left oid 30 (ObjNum 462 snap 121 seq_num 462) 2024-09-21T09:55:45.104 INFO:tasks.rados.rados.0.smithi033.stdout:1405: expect (ObjNum 455 snap 120 seq_num 455) 2024-09-21T09:55:45.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:44 smithi033 podman[103827]: 2024-09-21 09:55:44.918064508 +0000 UTC m=+0.349079506 container init 28dc5f57b5bceaa29c8a5b97d30675e84972fa356f0949756fbb7daa467c94eb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T09:55:45.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:44 smithi033 podman[103827]: 2024-09-21 09:55:44.923057737 +0000 UTC m=+0.354072742 container start 28dc5f57b5bceaa29c8a5b97d30675e84972fa356f0949756fbb7daa467c94eb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T09:55:45.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:44 smithi033 ceph-osd[103866]: -- 172.21.15.33:0/3446571504 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5565f93f5a40 con 0x5565fa1c4400 2024-09-21T09:55:45.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:45 smithi033 bash[103827]: 28dc5f57b5bceaa29c8a5b97d30675e84972fa356f0949756fbb7daa467c94eb 2024-09-21T09:55:45.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:45 smithi033 systemd[1]: Started Ceph osd.0 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:55:45.632 INFO:tasks.rados.rados.0.smithi033.stdout:1402: done (3 left) 2024-09-21T09:55:45.632 INFO:tasks.rados.rados.0.smithi033.stdout:1404: done (2 left) 2024-09-21T09:55:45.632 INFO:tasks.rados.rados.0.smithi033.stdout:1405: done (1 left) 2024-09-21T09:55:45.633 INFO:tasks.rados.rados.0.smithi033.stdout:1406: done (0 left) 2024-09-21T09:55:45.633 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 122 2024-09-21T09:55:45.633 INFO:tasks.rados.rados.0.smithi033.stdout:1407: seq_num 463 ranges {5660672=557056} 2024-09-21T09:55:45.666 INFO:tasks.rados.rados.0.smithi033.stdout:1407: writing smithi03385152-44 from 5660672 to 6217728 tid 1 2024-09-21T09:55:45.666 INFO:tasks.rados.rados.0.smithi033.stdout:1408: delete oid 45 current snap is 122 2024-09-21T09:55:45.672 INFO:tasks.rados.rados.0.smithi033.stdout:1408: done (1 left) 2024-09-21T09:55:45.672 INFO:tasks.rados.rados.0.smithi033.stdout:1409: snap_create 2024-09-21T09:55:45.675 INFO:tasks.rados.rados.0.smithi033.stdout:1407: finishing write tid 1 to smithi03385152-44 2024-09-21T09:55:45.675 INFO:tasks.rados.rados.0.smithi033.stdout:1407: finishing write tid 2 to smithi03385152-44 2024-09-21T09:55:45.676 INFO:tasks.rados.rados.0.smithi033.stdout:1407: finishing write tid 3 to smithi03385152-44 2024-09-21T09:55:45.676 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 692 (ObjNum 463 snap 122 seq_num 463) dirty exists 2024-09-21T09:55:45.676 INFO:tasks.rados.rados.0.smithi033.stdout:1407: left oid 44 (ObjNum 463 snap 122 seq_num 463) 2024-09-21T09:55:45.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:45 smithi105 ceph-mon[86649]: pgmap v105: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.4 MiB/s wr, 32 op/s; 66/312 objects degraded (21.154%) 2024-09-21T09:55:45.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:45 smithi105 ceph-mon[86649]: osdmap e318: 8 total, 7 up, 8 in 2024-09-21T09:55:45.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:45 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:45.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:45 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:45.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:45 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[98546]: pgmap v105: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.4 MiB/s wr, 32 op/s; 66/312 objects degraded (21.154%) 2024-09-21T09:55:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[98546]: osdmap e318: 8 total, 7 up, 8 in 2024-09-21T09:55:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[96568]: pgmap v105: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 230 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 7.7 MiB/s rd, 6.4 MiB/s wr, 32 op/s; 66/312 objects degraded (21.154%) 2024-09-21T09:55:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[96568]: osdmap e318: 8 total, 7 up, 8 in 2024-09-21T09:55:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:45.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:45.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:45 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:46.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:46.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:46.481 INFO:tasks.rados.rados.0.smithi033.stdout:1407: done (1 left) 2024-09-21T09:55:46.481 INFO:tasks.rados.rados.0.smithi033.stdout:1409: done (0 left) 2024-09-21T09:55:46.481 INFO:tasks.rados.rados.0.smithi033.stdout:1410: snap_remove snap 121 2024-09-21T09:55:46.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:46 smithi033 ceph-mon[98546]: osdmap e319: 8 total, 7 up, 8 in 2024-09-21T09:55:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:46.747 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:55:46.571+0000 7f2c7c4c4640 -1 Falling back to public interface 2024-09-21T09:55:46.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:46 smithi033 ceph-mon[96568]: osdmap e319: 8 total, 7 up, 8 in 2024-09-21T09:55:46.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:46.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:46 smithi105 ceph-mon[86649]: osdmap e319: 8 total, 7 up, 8 in 2024-09-21T09:55:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:55:47.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:46.950Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:47.482 INFO:tasks.rados.rados.0.smithi033.stdout:1410: done (0 left) 2024-09-21T09:55:47.482 INFO:tasks.rados.rados.0.smithi033.stdout:1411: snap_create 2024-09-21T09:55:47.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:47 smithi033 ceph-mon[98546]: pgmap v108: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.1 MiB/s wr, 26 op/s; 70/300 objects degraded (23.333%) 2024-09-21T09:55:47.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:47 smithi033 ceph-mon[98546]: osdmap e320: 8 total, 7 up, 8 in 2024-09-21T09:55:47.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:47 smithi033 ceph-mon[96568]: pgmap v108: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.1 MiB/s wr, 26 op/s; 70/300 objects degraded (23.333%) 2024-09-21T09:55:47.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:47 smithi033 ceph-mon[96568]: osdmap e320: 8 total, 7 up, 8 in 2024-09-21T09:55:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:47 smithi105 ceph-mon[86649]: pgmap v108: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.1 MiB/s wr, 26 op/s; 70/300 objects degraded (23.333%) 2024-09-21T09:55:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:47 smithi105 ceph-mon[86649]: osdmap e320: 8 total, 7 up, 8 in 2024-09-21T09:55:48.483 INFO:tasks.rados.rados.0.smithi033.stdout:1411: done (0 left) 2024-09-21T09:55:48.484 INFO:tasks.rados.rados.0.smithi033.stdout:1412: read oid 11 snap -1 2024-09-21T09:55:48.484 INFO:tasks.rados.rados.0.smithi033.stdout:1412: expect deleted 2024-09-21T09:55:48.484 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 124 2024-09-21T09:55:48.484 INFO:tasks.rados.rados.0.smithi033.stdout:1413: seq_num 465 ranges {11304960=598016} 2024-09-21T09:55:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:1413: writing smithi03385152-24 from 11304960 to 11902976 tid 1 2024-09-21T09:55:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 124 2024-09-21T09:55:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:1414: seq_num 466 ranges {475136=614400} 2024-09-21T09:55:48.555 INFO:tasks.rados.rados.0.smithi033.stdout:1414: writing smithi03385152-32 from 475136 to 1089536 tid 1 2024-09-21T09:55:48.555 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 124 2024-09-21T09:55:48.555 INFO:tasks.rados.rados.0.smithi033.stdout:1413: finishing write tid 1 to smithi03385152-24 2024-09-21T09:55:48.555 INFO:tasks.rados.rados.0.smithi033.stdout:1413: finishing write tid 2 to smithi03385152-24 2024-09-21T09:55:48.556 INFO:tasks.rados.rados.0.smithi033.stdout:1415: seq_num 467 ranges {0=630784} 2024-09-21T09:55:48.559 INFO:tasks.rados.rados.0.smithi033.stdout:1415: writing smithi03385152-36 from 0 to 630784 tid 1 2024-09-21T09:55:48.559 INFO:tasks.rados.rados.0.smithi033.stdout:1412: done (3 left) 2024-09-21T09:55:48.560 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 124 2024-09-21T09:55:48.560 INFO:tasks.rados.rados.0.smithi033.stdout:1413: finishing write tid 3 to smithi03385152-24 2024-09-21T09:55:48.560 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 649 (ObjNum 465 snap 124 seq_num 465) dirty exists 2024-09-21T09:55:48.560 INFO:tasks.rados.rados.0.smithi033.stdout:1413: left oid 24 (ObjNum 465 snap 124 seq_num 465) 2024-09-21T09:55:48.560 INFO:tasks.rados.rados.0.smithi033.stdout:1414: finishing write tid 1 to smithi03385152-32 2024-09-21T09:55:48.560 INFO:tasks.rados.rados.0.smithi033.stdout:1414: finishing write tid 2 to smithi03385152-32 2024-09-21T09:55:48.560 INFO:tasks.rados.rados.0.smithi033.stdout:1416: seq_num 468 ranges {0=647168} 2024-09-21T09:55:48.563 INFO:tasks.rados.rados.0.smithi033.stdout:1416: writing smithi03385152-45 from 0 to 647168 tid 1 2024-09-21T09:55:48.563 INFO:tasks.rados.rados.0.smithi033.stdout:1413: done (3 left) 2024-09-21T09:55:48.563 INFO:tasks.rados.rados.0.smithi033.stdout:1417: snap_remove snap 123 2024-09-21T09:55:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:48 smithi033 ceph-mon[98546]: osdmap e321: 8 total, 7 up, 8 in 2024-09-21T09:55:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:48 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 70/300 objects degraded (23.333%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:48 smithi033 ceph-mon[98546]: osdmap e322: 8 total, 7 up, 8 in 2024-09-21T09:55:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:48 smithi033 ceph-mon[96568]: osdmap e321: 8 total, 7 up, 8 in 2024-09-21T09:55:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:48 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 70/300 objects degraded (23.333%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:48.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:48 smithi033 ceph-mon[96568]: osdmap e322: 8 total, 7 up, 8 in 2024-09-21T09:55:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:48 smithi105 ceph-mon[86649]: osdmap e321: 8 total, 7 up, 8 in 2024-09-21T09:55:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:48 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 70/300 objects degraded (23.333%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:48 smithi105 ceph-mon[86649]: osdmap e322: 8 total, 7 up, 8 in 2024-09-21T09:55:49.502 INFO:tasks.rados.rados.0.smithi033.stdout:1417: done (3 left) 2024-09-21T09:55:49.502 INFO:tasks.rados.rados.0.smithi033.stdout:1418: rollback oid 20 current snap is 124 2024-09-21T09:55:49.503 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 122 2024-09-21T09:55:49.503 INFO:tasks.rados.rados.0.smithi033.stdout:1414: finishing write tid 3 to smithi03385152-32 2024-09-21T09:55:49.503 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 150 (ObjNum 466 snap 124 seq_num 466) dirty exists 2024-09-21T09:55:49.503 INFO:tasks.rados.rados.0.smithi033.stdout:1414: left oid 32 (ObjNum 466 snap 124 seq_num 466) 2024-09-21T09:55:49.503 INFO:tasks.rados.rados.0.smithi033.stdout:1415: finishing write tid 1 to smithi03385152-36 2024-09-21T09:55:49.503 INFO:tasks.rados.rados.0.smithi033.stdout:1415: finishing write tid 2 to smithi03385152-36 2024-09-21T09:55:49.503 INFO:tasks.rados.rados.0.smithi033.stdout:1415: finishing write tid 3 to smithi03385152-36 2024-09-21T09:55:49.504 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 888 (ObjNum 467 snap 124 seq_num 467) dirty exists 2024-09-21T09:55:49.504 INFO:tasks.rados.rados.0.smithi033.stdout:1415: left oid 36 (ObjNum 467 snap 124 seq_num 467) 2024-09-21T09:55:49.504 INFO:tasks.rados.rados.0.smithi033.stdout:1416: finishing write tid 1 to smithi03385152-45 2024-09-21T09:55:49.504 INFO:tasks.rados.rados.0.smithi033.stdout:1416: finishing write tid 2 to smithi03385152-45 2024-09-21T09:55:49.504 INFO:tasks.rados.rados.0.smithi033.stdout:1416: finishing write tid 3 to smithi03385152-45 2024-09-21T09:55:49.504 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 823 (ObjNum 468 snap 124 seq_num 468) dirty exists 2024-09-21T09:55:49.504 INFO:tasks.rados.rados.0.smithi033.stdout:1416: left oid 45 (ObjNum 468 snap 124 seq_num 468) 2024-09-21T09:55:49.505 INFO:tasks.rados.rados.0.smithi033.stdout:1414: done (3 left) 2024-09-21T09:55:49.505 INFO:tasks.rados.rados.0.smithi033.stdout:1415: done (2 left) 2024-09-21T09:55:49.505 INFO:tasks.rados.rados.0.smithi033.stdout:1416: done (1 left) 2024-09-21T09:55:49.505 INFO:tasks.rados.rados.0.smithi033.stdout:1419: delete oid 28 current snap is 124 2024-09-21T09:55:49.506 INFO:tasks.rados.rados.0.smithi033.stdout:1419: done (1 left) 2024-09-21T09:55:49.506 INFO:tasks.rados.rados.0.smithi033.stdout:1420: read oid 34 snap 122 2024-09-21T09:55:49.506 INFO:tasks.rados.rados.0.smithi033.stdout:1420: expect deleted 2024-09-21T09:55:49.506 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 124 2024-09-21T09:55:49.506 INFO:tasks.rados.rados.0.smithi033.stdout:1421: seq_num 470 ranges {5521408=679936} 2024-09-21T09:55:49.540 INFO:tasks.rados.rados.0.smithi033.stdout:1421: writing smithi03385152-17 from 5521408 to 6201344 tid 1 2024-09-21T09:55:49.540 INFO:tasks.rados.rados.0.smithi033.stdout:1422: copy_from oid 40 from oid 41 current snap is 124 2024-09-21T09:55:49.540 INFO:tasks.rados.rados.0.smithi033.stdout:1418: finishing rollback tid 1 to smithi03385152-20 2024-09-21T09:55:49.540 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 685 (ObjNum 1665 snap 0 seq_num 1163895152) dirty dne 2024-09-21T09:55:49.540 INFO:tasks.rados.rados.0.smithi033.stdout:1418: done (3 left) 2024-09-21T09:55:49.540 INFO:tasks.rados.rados.0.smithi033.stdout:1420: done (2 left) 2024-09-21T09:55:49.540 INFO:tasks.rados.rados.0.smithi033.stdout:1423: snap_remove snap 122 2024-09-21T09:55:49.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:49 smithi033 ceph-mon[96568]: pgmap v111: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.1 MiB/s wr, 26 op/s; 70/300 objects degraded (23.333%) 2024-09-21T09:55:49.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:49 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:49.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:49 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:49 smithi033 ceph-mon[98546]: pgmap v111: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.1 MiB/s wr, 26 op/s; 70/300 objects degraded (23.333%) 2024-09-21T09:55:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:49 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:49 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:49 smithi105 ceph-mon[86649]: pgmap v111: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 9.0 MiB/s rd, 1.1 MiB/s wr, 26 op/s; 70/300 objects degraded (23.333%) 2024-09-21T09:55:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:49 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:49 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:50.381 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:55:50.229+0000 7f2c7c4c4640 -1 osd.0 312 log_to_monitors true 2024-09-21T09:55:50.500 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1423: done (2 left) 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1424: snap_create 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1421: finishing write tid 1 to smithi03385152-17 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1421: finishing write tid 2 to smithi03385152-17 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1421: finishing write tid 3 to smithi03385152-17 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 535 (ObjNum 470 snap 124 seq_num 470) dirty exists 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1421: left oid 17 (ObjNum 470 snap 124 seq_num 470) 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1422: finishing copy_from racing read to smithi03385152-40 2024-09-21T09:55:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1422: finishing copy_from to smithi03385152-40 2024-09-21T09:55:50.509 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 701 (ObjNum 459 snap 121 seq_num 459) dirty exists 2024-09-21T09:55:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-mon[96568]: osdmap e323: 8 total, 7 up, 8 in 2024-09-21T09:55:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-mon[96568]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:55:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-mon[98546]: osdmap e323: 8 total, 7 up, 8 in 2024-09-21T09:55:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:50.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-mon[98546]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:55:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:50 smithi105 ceph-mon[86649]: osdmap e323: 8 total, 7 up, 8 in 2024-09-21T09:55:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:50.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:50 smithi105 ceph-mon[86649]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-21T09:55:51.193 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 09:55:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T09:55:50.972+0000 7f2c70266640 -1 osd.0 312 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:55:51.518 INFO:tasks.rados.rados.0.smithi033.stdout:1421: done (2 left) 2024-09-21T09:55:51.518 INFO:tasks.rados.rados.0.smithi033.stdout:1422: done (1 left) 2024-09-21T09:55:51.519 INFO:tasks.rados.rados.0.smithi033.stdout:1424: done (0 left) 2024-09-21T09:55:51.519 INFO:tasks.rados.rados.0.smithi033.stdout:1425: read oid 28 snap -1 2024-09-21T09:55:51.519 INFO:tasks.rados.rados.0.smithi033.stdout:1425: expect deleted 2024-09-21T09:55:51.519 INFO:tasks.rados.rados.0.smithi033.stdout:1426: read oid 11 snap -1 2024-09-21T09:55:51.519 INFO:tasks.rados.rados.0.smithi033.stdout:1426: expect deleted 2024-09-21T09:55:51.519 INFO:tasks.rados.rados.0.smithi033.stdout:1427: read oid 14 snap -1 2024-09-21T09:55:51.519 INFO:tasks.rados.rados.0.smithi033.stdout:1427: expect (ObjNum 455 snap 120 seq_num 455) 2024-09-21T09:55:51.519 INFO:tasks.rados.rados.0.smithi033.stdout:1428: delete oid 42 current snap is 125 2024-09-21T09:55:51.555 INFO:tasks.rados.rados.0.smithi033.stdout:1427: expect (ObjNum 455 snap 120 seq_num 455) 2024-09-21T09:55:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[98546]: pgmap v114: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 70/300 objects degraded (23.333%) 2024-09-21T09:55:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[98546]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:55:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[98546]: osdmap e324: 8 total, 7 up, 8 in 2024-09-21T09:55:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[98546]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:55:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[98546]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' 2024-09-21T09:55:51.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:55:51] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.51.0" 2024-09-21T09:55:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[96568]: pgmap v114: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 70/300 objects degraded (23.333%) 2024-09-21T09:55:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[96568]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:55:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[96568]: osdmap e324: 8 total, 7 up, 8 in 2024-09-21T09:55:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[96568]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:55:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:51 smithi033 ceph-mon[96568]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' 2024-09-21T09:55:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:51 smithi105 ceph-mon[86649]: pgmap v114: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 211 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 70/300 objects degraded (23.333%) 2024-09-21T09:55:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:51 smithi105 ceph-mon[86649]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-21T09:55:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:51 smithi105 ceph-mon[86649]: osdmap e324: 8 total, 7 up, 8 in 2024-09-21T09:55:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:51 smithi105 ceph-mon[86649]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:55:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:51 smithi105 ceph-mon[86649]: from='osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874]' entity='osd.0' 2024-09-21T09:55:52.559 INFO:tasks.rados.rados.0.smithi033.stdout:1425: done (3 left) 2024-09-21T09:55:52.559 INFO:tasks.rados.rados.0.smithi033.stdout:1426: done (2 left) 2024-09-21T09:55:52.559 INFO:tasks.rados.rados.0.smithi033.stdout:1427: done (1 left) 2024-09-21T09:55:52.559 INFO:tasks.rados.rados.0.smithi033.stdout:1428: done (0 left) 2024-09-21T09:55:52.559 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 125 2024-09-21T09:55:52.559 INFO:tasks.rados.rados.0.smithi033.stdout:1429: seq_num 472 ranges {5758976=712704} 2024-09-21T09:55:52.596 INFO:tasks.rados.rados.0.smithi033.stdout:1429: writing smithi03385152-8 from 5758976 to 6471680 tid 1 2024-09-21T09:55:52.596 INFO:tasks.rados.rados.0.smithi033.stdout:1430: delete oid 16 current snap is 125 2024-09-21T09:55:52.599 INFO:tasks.rados.rados.0.smithi033.stdout:1430: done (1 left) 2024-09-21T09:55:52.599 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 125 2024-09-21T09:55:52.599 INFO:tasks.rados.rados.0.smithi033.stdout:1431: seq_num 474 ranges {0=745472} 2024-09-21T09:55:52.604 INFO:tasks.rados.rados.0.smithi033.stdout:1431: writing smithi03385152-7 from 0 to 745472 tid 1 2024-09-21T09:55:52.604 INFO:tasks.rados.rados.0.smithi033.stdout:1432: rollback oid 47 current snap is 125 2024-09-21T09:55:52.604 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 124 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1429: finishing write tid 1 to smithi03385152-8 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1429: finishing write tid 2 to smithi03385152-8 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1429: finishing write tid 3 to smithi03385152-8 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 643 (ObjNum 472 snap 125 seq_num 472) dirty exists 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1429: left oid 8 (ObjNum 472 snap 125 seq_num 472) 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1429: done (2 left) 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1433: copy_from oid 29 from oid 11 current snap is 125 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1434: read oid 50 snap -1 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1434: expect (ObjNum 402 snap 110 seq_num 402) 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1435: rmattr oid 1 current snap is 125 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1435: done (4 left) 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1436: read oid 2 snap -1 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1436: expect (ObjNum 400 snap 108 seq_num 400) 2024-09-21T09:55:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:1437: delete oid 19 current snap is 125 2024-09-21T09:55:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:1437: done (5 left) 2024-09-21T09:55:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:1438: rollback oid 33 current snap is 125 2024-09-21T09:55:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 124 2024-09-21T09:55:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 125 2024-09-21T09:55:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:1439: seq_num 476 ranges {1015808=778240} 2024-09-21T09:55:52.617 INFO:tasks.rados.rados.0.smithi033.stdout:1439: writing smithi03385152-49 from 1015808 to 1794048 tid 1 2024-09-21T09:55:52.617 INFO:tasks.rados.rados.0.smithi033.stdout:1440: setattr oid 42 current snap is 125 2024-09-21T09:55:52.617 INFO:tasks.rados.rados.0.smithi033.stdout:1431: finishing write tid 1 to smithi03385152-7 2024-09-21T09:55:52.617 INFO:tasks.rados.rados.0.smithi033.stdout:1438: finishing rollback tid 0 to smithi03385152-33 2024-09-21T09:55:52.617 INFO:tasks.rados.rados.0.smithi033.stdout:1431: finishing write tid 2 to smithi03385152-7 2024-09-21T09:55:52.617 INFO:tasks.rados.rados.0.smithi033.stdout:1438: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:55:52.617 INFO:tasks.rados.rados.0.smithi033.stdout:1438: finishing rollback tid 2 to smithi03385152-33 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 482 (ObjNum 409 snap 110 seq_num 409) dirty exists 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:1431: finishing write tid 3 to smithi03385152-7 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 892 (ObjNum 474 snap 125 seq_num 474) dirty exists 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:1431: left oid 7 (ObjNum 474 snap 125 seq_num 474) 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:1433: finishing copy_from to smithi03385152-29 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:1433: got expected ENOENT (src dne) 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:1433: finishing copy_from racing read to smithi03385152-29 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:1432: finishing rollback tid 0 to smithi03385152-47 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:1432: finishing rollback tid 1 to smithi03385152-47 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:1432: finishing rollback tid 2 to smithi03385152-47 2024-09-21T09:55:52.618 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 645 (ObjNum 279 snap 69 seq_num 279) dirty exists 2024-09-21T09:55:52.619 INFO:tasks.rados.rados.0.smithi033.stdout:1431: done (7 left) 2024-09-21T09:55:52.619 INFO:tasks.rados.rados.0.smithi033.stdout:1432: done (6 left) 2024-09-21T09:55:52.619 INFO:tasks.rados.rados.0.smithi033.stdout:1433: done (5 left) 2024-09-21T09:55:52.619 INFO:tasks.rados.rados.0.smithi033.stdout:1438: done (4 left) 2024-09-21T09:55:52.619 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 125 2024-09-21T09:55:52.619 INFO:tasks.rados.rados.0.smithi033.stdout:1441: seq_num 477 ranges {2465792=794624} 2024-09-21T09:55:52.637 INFO:tasks.rados.rados.0.smithi033.stdout:1441: writing smithi03385152-41 from 2465792 to 3260416 tid 1 2024-09-21T09:55:52.638 INFO:tasks.rados.rados.0.smithi033.stdout:1442: copy_from oid 31 from oid 35 current snap is 125 2024-09-21T09:55:52.638 INFO:tasks.rados.rados.0.smithi033.stdout:1434: expect (ObjNum 402 snap 110 seq_num 402) 2024-09-21T09:55:52.694 INFO:tasks.rados.rados.0.smithi033.stdout:1439: finishing write tid 1 to smithi03385152-49 2024-09-21T09:55:52.694 INFO:tasks.rados.rados.0.smithi033.stdout:1439: finishing write tid 2 to smithi03385152-49 2024-09-21T09:55:52.695 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 704 (ObjNum 241 snap 0 seq_num 1163894208) dirty exists 2024-09-21T09:55:52.695 INFO:tasks.rados.rados.0.smithi033.stdout:1439: finishing write tid 3 to smithi03385152-49 2024-09-21T09:55:52.695 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 540 (ObjNum 476 snap 125 seq_num 476) dirty exists 2024-09-21T09:55:52.695 INFO:tasks.rados.rados.0.smithi033.stdout:1439: left oid 49 (ObjNum 476 snap 125 seq_num 476) 2024-09-21T09:55:52.695 INFO:tasks.rados.rados.0.smithi033.stdout:1436: expect (ObjNum 400 snap 108 seq_num 400) 2024-09-21T09:55:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:52 smithi105 ceph-mon[86649]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:55:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:52 smithi105 ceph-mon[86649]: osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874] boot 2024-09-21T09:55:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:52 smithi105 ceph-mon[86649]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:55:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:55:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:52 smithi105 ceph-mon[86649]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1442: finishing copy_from racing read to smithi03385152-31 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1442: finishing copy_from to smithi03385152-31 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 834 (ObjNum 435 snap 115 seq_num 435) dirty exists 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1441: finishing write tid 1 to smithi03385152-41 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1441: finishing write tid 2 to smithi03385152-41 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1441: finishing write tid 3 to smithi03385152-41 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 706 (ObjNum 477 snap 125 seq_num 477) dirty exists 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1441: left oid 41 (ObjNum 477 snap 125 seq_num 477) 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1434: done (5 left) 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1436: done (4 left) 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1439: done (3 left) 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1440: done (2 left) 2024-09-21T09:55:52.832 INFO:tasks.rados.rados.0.smithi033.stdout:1441: done (1 left) 2024-09-21T09:55:52.833 INFO:tasks.rados.rados.0.smithi033.stdout:1442: done (0 left) 2024-09-21T09:55:52.833 INFO:tasks.rados.rados.0.smithi033.stdout:1443: read oid 12 snap 124 2024-09-21T09:55:52.833 INFO:tasks.rados.rados.0.smithi033.stdout:1443: expect (ObjNum 411 snap 110 seq_num 411) 2024-09-21T09:55:52.833 INFO:tasks.rados.rados.0.smithi033.stdout:1444: rmattr oid 33 current snap is 125 2024-09-21T09:55:52.833 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 125 2024-09-21T09:55:52.833 INFO:tasks.rados.rados.0.smithi033.stdout:1445: seq_num 478 ranges {11902976=409600,12312576=401408} 2024-09-21T09:55:52.900 INFO:tasks.rados.rados.0.smithi033.stdout:1445: writing smithi03385152-24 from 11902976 to 12312576 tid 1 2024-09-21T09:55:52.901 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[98546]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:55:52.901 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[98546]: osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874] boot 2024-09-21T09:55:52.901 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[98546]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:55:52.901 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:55:52.901 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[98546]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:55:52.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[96568]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:55:52.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[96568]: osd.0 [v2:172.21.15.33:6802/189402874,v1:172.21.15.33:6803/189402874] boot 2024-09-21T09:55:52.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[96568]: osdmap e325: 8 total, 8 up, 8 in 2024-09-21T09:55:52.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-21T09:55:52.902 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:52 smithi033 ceph-mon[96568]: osdmap e326: 8 total, 8 up, 8 in 2024-09-21T09:55:52.903 INFO:tasks.rados.rados.0.smithi033.stdout:1445: writing smithi03385152-24 from 12312576 to 12713984 tid 2 2024-09-21T09:55:52.904 INFO:tasks.rados.rados.0.smithi033.stdout:1446: read oid 41 snap -1 2024-09-21T09:55:52.904 INFO:tasks.rados.rados.0.smithi033.stdout:1446: expect (ObjNum 477 snap 125 seq_num 477) 2024-09-21T09:55:52.904 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 125 2024-09-21T09:55:52.904 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 483 (ObjNum 409 snap 110 seq_num 409) dirty exists 2024-09-21T09:55:52.904 INFO:tasks.rados.rados.0.smithi033.stdout:1447: seq_num 479 ranges {475136=425984,901120=401408} 2024-09-21T09:55:52.909 INFO:tasks.rados.rados.0.smithi033.stdout:1447: writing smithi03385152-9 from 475136 to 901120 tid 1 2024-09-21T09:55:52.911 INFO:tasks.rados.rados.0.smithi033.stdout:1447: writing smithi03385152-9 from 901120 to 1302528 tid 2 2024-09-21T09:55:52.912 INFO:tasks.rados.rados.0.smithi033.stdout:1444: done (4 left) 2024-09-21T09:55:52.912 INFO:tasks.rados.rados.0.smithi033.stdout:1448: rollback oid 37 current snap is 125 2024-09-21T09:55:52.912 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 124 2024-09-21T09:55:52.912 INFO:tasks.rados.rados.0.smithi033.stdout:1443: expect (ObjNum 411 snap 110 seq_num 411) 2024-09-21T09:55:52.973 INFO:tasks.rados.rados.0.smithi033.stdout:1445: finishing write tid 1 to smithi03385152-24 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:1445: finishing write tid 2 to smithi03385152-24 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:1445: finishing write tid 3 to smithi03385152-24 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:1445: finishing write tid 4 to smithi03385152-24 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 652 (ObjNum 478 snap 125 seq_num 478) dirty exists 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:1445: left oid 24 (ObjNum 478 snap 125 seq_num 478) 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:1443: done (4 left) 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:1445: done (3 left) 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:1449: rollback oid 1 current snap is 125 2024-09-21T09:55:52.974 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 124 2024-09-21T09:55:52.975 INFO:tasks.rados.rados.0.smithi033.stdout:1448: finishing rollback tid 0 to smithi03385152-37 2024-09-21T09:55:52.975 INFO:tasks.rados.rados.0.smithi033.stdout:1448: finishing rollback tid 1 to smithi03385152-37 2024-09-21T09:55:52.975 INFO:tasks.rados.rados.0.smithi033.stdout:1448: finishing rollback tid 2 to smithi03385152-37 2024-09-21T09:55:52.975 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 707 (ObjNum 427 snap 113 seq_num 427) dirty exists 2024-09-21T09:55:52.975 INFO:tasks.rados.rados.0.smithi033.stdout:1446: expect (ObjNum 477 snap 125 seq_num 477) 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:1447: finishing write tid 1 to smithi03385152-9 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:1447: finishing write tid 2 to smithi03385152-9 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:1447: finishing write tid 3 to smithi03385152-9 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:1447: finishing write tid 4 to smithi03385152-9 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 709 (ObjNum 479 snap 125 seq_num 479) dirty exists 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:1447: left oid 9 (ObjNum 479 snap 125 seq_num 479) 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:1446: done (3 left) 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:1447: done (2 left) 2024-09-21T09:55:53.137 INFO:tasks.rados.rados.0.smithi033.stdout:1448: done (1 left) 2024-09-21T09:55:53.138 INFO:tasks.rados.rados.0.smithi033.stdout:1450: rmattr oid 30 current snap is 125 2024-09-21T09:55:53.138 INFO:tasks.rados.rados.0.smithi033.stdout:1449: finishing rollback tid 0 to smithi03385152-1 2024-09-21T09:55:53.138 INFO:tasks.rados.rados.0.smithi033.stdout:1449: finishing rollback tid 1 to smithi03385152-1 2024-09-21T09:55:53.138 INFO:tasks.rados.rados.0.smithi033.stdout:1449: finishing rollback tid 2 to smithi03385152-1 2024-09-21T09:55:53.138 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 646 (ObjNum 404 snap 110 seq_num 404) dirty exists 2024-09-21T09:55:53.138 INFO:tasks.rados.rados.0.smithi033.stdout:1449: done (1 left) 2024-09-21T09:55:53.138 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 125 2024-09-21T09:55:53.138 INFO:tasks.rados.rados.0.smithi033.stdout:1451: seq_num 480 ranges {2097152=442368,2539520=401408} 2024-09-21T09:55:53.150 INFO:tasks.rados.rados.0.smithi033.stdout:1451: writing smithi03385152-12 from 2097152 to 2539520 tid 1 2024-09-21T09:55:53.153 INFO:tasks.rados.rados.0.smithi033.stdout:1451: writing smithi03385152-12 from 2539520 to 2940928 tid 2 2024-09-21T09:55:53.153 INFO:tasks.rados.rados.0.smithi033.stdout:1452: read oid 14 snap -1 2024-09-21T09:55:53.153 INFO:tasks.rados.rados.0.smithi033.stdout:1452: expect (ObjNum 455 snap 120 seq_num 455) 2024-09-21T09:55:53.153 INFO:tasks.rados.rados.0.smithi033.stdout:1453: rollback oid 18 current snap is 125 2024-09-21T09:55:53.153 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 124 2024-09-21T09:55:53.153 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 918 (ObjNum 462 snap 121 seq_num 462) dirty exists 2024-09-21T09:55:53.154 INFO:tasks.rados.rados.0.smithi033.stdout:1450: done (3 left) 2024-09-21T09:55:53.154 INFO:tasks.rados.rados.0.smithi033.stdout:1454: snap_create 2024-09-21T09:55:53.154 INFO:tasks.rados.rados.0.smithi033.stdout:1451: finishing write tid 1 to smithi03385152-12 2024-09-21T09:55:53.154 INFO:tasks.rados.rados.0.smithi033.stdout:1453: finishing rollback tid 1 to smithi03385152-18 2024-09-21T09:55:53.154 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 0 (ObjNum 33 snap 0 seq_num 90971744) dirty dne 2024-09-21T09:55:53.156 INFO:tasks.rados.rados.0.smithi033.stdout:1451: finishing write tid 2 to smithi03385152-12 2024-09-21T09:55:53.156 INFO:tasks.rados.rados.0.smithi033.stdout:1451: finishing write tid 3 to smithi03385152-12 2024-09-21T09:55:53.157 INFO:tasks.rados.rados.0.smithi033.stdout:1451: finishing write tid 4 to smithi03385152-12 2024-09-21T09:55:53.157 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 544 (ObjNum 480 snap 125 seq_num 480) dirty exists 2024-09-21T09:55:53.157 INFO:tasks.rados.rados.0.smithi033.stdout:1451: left oid 12 (ObjNum 480 snap 125 seq_num 480) 2024-09-21T09:55:53.177 INFO:tasks.rados.rados.0.smithi033.stdout:1452: expect (ObjNum 455 snap 120 seq_num 455) 2024-09-21T09:55:53.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:55:53.041+0000 7fc38e955640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:55:53.960 INFO:tasks.rados.rados.0.smithi033.stdout:1451: done (3 left) 2024-09-21T09:55:53.960 INFO:tasks.rados.rados.0.smithi033.stdout:1452: done (2 left) 2024-09-21T09:55:53.960 INFO:tasks.rados.rados.0.smithi033.stdout:1453: done (1 left) 2024-09-21T09:55:53.960 INFO:tasks.rados.rados.0.smithi033.stdout:1454: done (0 left) 2024-09-21T09:55:53.961 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 126 2024-09-21T09:55:53.961 INFO:tasks.rados.rados.0.smithi033.stdout:1455: seq_num 481 ranges {0=458752,458752=401408} 2024-09-21T09:55:53.962 INFO:tasks.rados.rados.0.smithi033.stdout:1455: writing smithi03385152-25 from 0 to 458752 tid 1 2024-09-21T09:55:53.965 INFO:tasks.rados.rados.0.smithi033.stdout:1455: writing smithi03385152-25 from 458752 to 860160 tid 2 2024-09-21T09:55:53.965 INFO:tasks.rados.rados.0.smithi033.stdout:1456: delete oid 33 current snap is 126 2024-09-21T09:55:53.969 INFO:tasks.rados.rados.0.smithi033.stdout:1455: finishing write tid 1 to smithi03385152-25 2024-09-21T09:55:53.970 INFO:tasks.rados.rados.0.smithi033.stdout:1456: done (1 left) 2024-09-21T09:55:53.970 INFO:tasks.rados.rados.0.smithi033.stdout:1457: snap_create 2024-09-21T09:55:53.970 INFO:tasks.rados.rados.0.smithi033.stdout:1455: finishing write tid 2 to smithi03385152-25 2024-09-21T09:55:53.971 INFO:tasks.rados.rados.0.smithi033.stdout:1455: finishing write tid 3 to smithi03385152-25 2024-09-21T09:55:53.972 INFO:tasks.rados.rados.0.smithi033.stdout:1455: finishing write tid 4 to smithi03385152-25 2024-09-21T09:55:53.972 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 636 (ObjNum 481 snap 126 seq_num 481) dirty exists 2024-09-21T09:55:53.972 INFO:tasks.rados.rados.0.smithi033.stdout:1455: left oid 25 (ObjNum 481 snap 126 seq_num 481) 2024-09-21T09:55:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: pgmap v117: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 212 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 72/303 objects degraded (23.762%) 2024-09-21T09:55:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 72/303 objects degraded (23.762%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: pgmap v117: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 212 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 72/303 objects degraded (23.762%) 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:53.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:53.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:53.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:53.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:55:53.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:53 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 72/303 objects degraded (23.762%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: pgmap v117: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 212 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 72/303 objects degraded (23.762%) 2024-09-21T09:55:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:55:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:55:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:55:54.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:55:54.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:54.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:54.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:55:54.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:55:54.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:53 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 72/303 objects degraded (23.762%), 16 pgs degraded (PG_DEGRADED) 2024-09-21T09:55:54.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:54.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:55:54.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:54.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:54.968 INFO:tasks.rados.rados.0.smithi033.stdout:1455: done (1 left) 2024-09-21T09:55:54.968 INFO:tasks.rados.rados.0.smithi033.stdout:1457: done (0 left) 2024-09-21T09:55:54.968 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 127 2024-09-21T09:55:54.968 INFO:tasks.rados.rados.0.smithi033.stdout:1458: seq_num 483 ranges {4554752=491520,5046272=401408} 2024-09-21T09:55:54.995 INFO:tasks.rados.rados.0.smithi033.stdout:1458: writing smithi03385152-5 from 4554752 to 5046272 tid 1 2024-09-21T09:55:54.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:54 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:55:54.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:54 smithi033 ceph-mon[96568]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:55:54.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:54 smithi033 ceph-mon[96568]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:55:54.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:54 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:55:54.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:54 smithi033 ceph-mon[98546]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:55:54.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:54 smithi033 ceph-mon[98546]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:55:54.997 INFO:tasks.rados.rados.0.smithi033.stdout:1458: writing smithi03385152-5 from 5046272 to 5447680 tid 2 2024-09-21T09:55:54.998 INFO:tasks.rados.rados.0.smithi033.stdout:1459: rollback oid 11 current snap is 127 2024-09-21T09:55:54.998 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 11 to 126 2024-09-21T09:55:54.998 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 127 2024-09-21T09:55:54.998 INFO:tasks.rados.rados.0.smithi033.stdout:1460: seq_num 484 ranges {2768896=507904,3276800=401408} 2024-09-21T09:55:55.016 INFO:tasks.rados.rados.0.smithi033.stdout:1460: writing smithi03385152-4 from 2768896 to 3276800 tid 1 2024-09-21T09:55:55.019 INFO:tasks.rados.rados.0.smithi033.stdout:1460: writing smithi03385152-4 from 3276800 to 3678208 tid 2 2024-09-21T09:55:55.019 INFO:tasks.rados.rados.0.smithi033.stdout:1461: delete oid 30 current snap is 127 2024-09-21T09:55:55.019 INFO:tasks.rados.rados.0.smithi033.stdout:1459: finishing rollback tid 1 to smithi03385152-11 2024-09-21T09:55:55.019 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:55:55.020 INFO:tasks.rados.rados.0.smithi033.stdout:1458: finishing write tid 1 to smithi03385152-5 2024-09-21T09:55:55.020 INFO:tasks.rados.rados.0.smithi033.stdout:1458: finishing write tid 2 to smithi03385152-5 2024-09-21T09:55:55.020 INFO:tasks.rados.rados.0.smithi033.stdout:1458: finishing write tid 3 to smithi03385152-5 2024-09-21T09:55:55.020 INFO:tasks.rados.rados.0.smithi033.stdout:1458: finishing write tid 4 to smithi03385152-5 2024-09-21T09:55:55.020 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 191 (ObjNum 483 snap 127 seq_num 483) dirty exists 2024-09-21T09:55:55.020 INFO:tasks.rados.rados.0.smithi033.stdout:1458: left oid 5 (ObjNum 483 snap 127 seq_num 483) 2024-09-21T09:55:55.022 INFO:tasks.rados.rados.0.smithi033.stdout:1458: done (3 left) 2024-09-21T09:55:55.022 INFO:tasks.rados.rados.0.smithi033.stdout:1459: done (2 left) 2024-09-21T09:55:55.022 INFO:tasks.rados.rados.0.smithi033.stdout:1461: done (1 left) 2024-09-21T09:55:55.022 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 127 2024-09-21T09:55:55.022 INFO:tasks.rados.rados.0.smithi033.stdout:1462: seq_num 486 ranges {1253376=548864,1802240=401408} 2024-09-21T09:55:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:54 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:55:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:54 smithi105 ceph-mon[86649]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:55:55.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:54 smithi105 ceph-mon[86649]: osdmap e327: 8 total, 8 up, 8 in 2024-09-21T09:55:55.032 INFO:tasks.rados.rados.0.smithi033.stdout:1462: writing smithi03385152-15 from 1253376 to 1802240 tid 1 2024-09-21T09:55:55.034 INFO:tasks.rados.rados.0.smithi033.stdout:1462: writing smithi03385152-15 from 1802240 to 2203648 tid 2 2024-09-21T09:55:55.034 INFO:tasks.rados.rados.0.smithi033.stdout:1463: delete oid 34 current snap is 127 2024-09-21T09:55:55.034 INFO:tasks.rados.rados.0.smithi033.stdout:1460: finishing write tid 1 to smithi03385152-4 2024-09-21T09:55:55.034 INFO:tasks.rados.rados.0.smithi033.stdout:1460: finishing write tid 2 to smithi03385152-4 2024-09-21T09:55:55.034 INFO:tasks.rados.rados.0.smithi033.stdout:1460: finishing write tid 3 to smithi03385152-4 2024-09-21T09:55:55.035 INFO:tasks.rados.rados.0.smithi033.stdout:1460: finishing write tid 4 to smithi03385152-4 2024-09-21T09:55:55.035 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 695 (ObjNum 484 snap 127 seq_num 484) dirty exists 2024-09-21T09:55:55.035 INFO:tasks.rados.rados.0.smithi033.stdout:1460: left oid 4 (ObjNum 484 snap 127 seq_num 484) 2024-09-21T09:55:55.036 INFO:tasks.rados.rados.0.smithi033.stdout:1460: done (2 left) 2024-09-21T09:55:55.036 INFO:tasks.rados.rados.0.smithi033.stdout:1463: done (1 left) 2024-09-21T09:55:55.037 INFO:tasks.rados.rados.0.smithi033.stdout:1464: rollback oid 13 current snap is 127 2024-09-21T09:55:55.037 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 13 to 126 2024-09-21T09:55:55.037 INFO:tasks.rados.rados.0.smithi033.stdout:1465: snap_create 2024-09-21T09:55:55.037 INFO:tasks.rados.rados.0.smithi033.stdout:1462: finishing write tid 1 to smithi03385152-15 2024-09-21T09:55:55.037 INFO:tasks.rados.rados.0.smithi033.stdout:1462: finishing write tid 2 to smithi03385152-15 2024-09-21T09:55:55.037 INFO:tasks.rados.rados.0.smithi033.stdout:1462: finishing write tid 3 to smithi03385152-15 2024-09-21T09:55:55.038 INFO:tasks.rados.rados.0.smithi033.stdout:1462: finishing write tid 4 to smithi03385152-15 2024-09-21T09:55:55.038 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 649 (ObjNum 486 snap 127 seq_num 486) dirty exists 2024-09-21T09:55:55.038 INFO:tasks.rados.rados.0.smithi033.stdout:1462: left oid 15 (ObjNum 486 snap 127 seq_num 486) 2024-09-21T09:55:55.040 INFO:tasks.rados.rados.0.smithi033.stdout:1464: finishing rollback tid 0 to smithi03385152-13 2024-09-21T09:55:55.040 INFO:tasks.rados.rados.0.smithi033.stdout:1464: finishing rollback tid 1 to smithi03385152-13 2024-09-21T09:55:55.041 INFO:tasks.rados.rados.0.smithi033.stdout:1464: finishing rollback tid 2 to smithi03385152-13 2024-09-21T09:55:55.041 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 924 (ObjNum 414 snap 110 seq_num 414) dirty exists 2024-09-21T09:55:55.970 INFO:tasks.rados.rados.0.smithi033.stdout:1462: done (2 left) 2024-09-21T09:55:55.970 INFO:tasks.rados.rados.0.smithi033.stdout:1464: done (1 left) 2024-09-21T09:55:55.970 INFO:tasks.rados.rados.0.smithi033.stdout:1465: done (0 left) 2024-09-21T09:55:55.970 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 128 2024-09-21T09:55:55.971 INFO:tasks.rados.rados.0.smithi033.stdout:1466: seq_num 488 ranges {4718592=581632,5300224=401408} 2024-09-21T09:55:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:55 smithi033 ceph-mon[98546]: pgmap v120: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 212 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 72/303 objects degraded (23.762%) 2024-09-21T09:55:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:55 smithi033 ceph-mon[98546]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:55:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:55 smithi033 ceph-mon[96568]: pgmap v120: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 212 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 72/303 objects degraded (23.762%) 2024-09-21T09:55:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:55 smithi033 ceph-mon[96568]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:55:55.999 INFO:tasks.rados.rados.0.smithi033.stdout:1466: writing smithi03385152-38 from 4718592 to 5300224 tid 1 2024-09-21T09:55:56.002 INFO:tasks.rados.rados.0.smithi033.stdout:1466: writing smithi03385152-38 from 5300224 to 5701632 tid 2 2024-09-21T09:55:56.002 INFO:tasks.rados.rados.0.smithi033.stdout:1467: rollback oid 50 current snap is 128 2024-09-21T09:55:56.002 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 50 to 127 2024-09-21T09:55:56.002 INFO:tasks.rados.rados.0.smithi033.stdout:1468: setattr oid 41 current snap is 128 2024-09-21T09:55:56.003 INFO:tasks.rados.rados.0.smithi033.stdout:1469: rollback oid 7 current snap is 128 2024-09-21T09:55:56.003 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 7 to 126 2024-09-21T09:55:56.004 INFO:tasks.rados.rados.0.smithi033.stdout:1470: snap_remove snap 125 2024-09-21T09:55:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:55 smithi105 ceph-mon[86649]: pgmap v120: 121 pgs: 37 active+undersized, 16 active+undersized+degraded, 68 active+clean; 212 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 72/303 objects degraded (23.762%) 2024-09-21T09:55:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:55 smithi105 ceph-mon[86649]: osdmap e328: 8 total, 8 up, 8 in 2024-09-21T09:55:56.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:55.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:56.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:56.984 INFO:tasks.rados.rados.0.smithi033.stdout:1470: done (4 left) 2024-09-21T09:55:56.984 INFO:tasks.rados.rados.0.smithi033.stdout:1471: rollback oid 14 current snap is 128 2024-09-21T09:55:56.984 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 14 to 126 2024-09-21T09:55:56.984 INFO:tasks.rados.rados.0.smithi033.stdout:1466: finishing write tid 1 to smithi03385152-38 2024-09-21T09:55:56.984 INFO:tasks.rados.rados.0.smithi033.stdout:1466: finishing write tid 2 to smithi03385152-38 2024-09-21T09:55:56.984 INFO:tasks.rados.rados.0.smithi033.stdout:1466: finishing write tid 3 to smithi03385152-38 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:1469: finishing rollback tid 0 to smithi03385152-7 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:1467: finishing rollback tid 0 to smithi03385152-50 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:1469: finishing rollback tid 1 to smithi03385152-7 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:1467: finishing rollback tid 1 to smithi03385152-50 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 713 (ObjNum 477 snap 125 seq_num 477) dirty exists 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:1469: finishing rollback tid 2 to smithi03385152-7 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 928 (ObjNum 474 snap 125 seq_num 474) dirty exists 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:1467: finishing rollback tid 2 to smithi03385152-50 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 639 (ObjNum 402 snap 110 seq_num 402) dirty exists 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:1466: finishing write tid 4 to smithi03385152-38 2024-09-21T09:55:56.985 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 712 (ObjNum 488 snap 128 seq_num 488) dirty exists 2024-09-21T09:55:56.986 INFO:tasks.rados.rados.0.smithi033.stdout:1466: left oid 38 (ObjNum 488 snap 128 seq_num 488) 2024-09-21T09:55:56.986 INFO:tasks.rados.rados.0.smithi033.stdout:1466: done (4 left) 2024-09-21T09:55:56.986 INFO:tasks.rados.rados.0.smithi033.stdout:1467: done (3 left) 2024-09-21T09:55:56.986 INFO:tasks.rados.rados.0.smithi033.stdout:1468: done (2 left) 2024-09-21T09:55:56.986 INFO:tasks.rados.rados.0.smithi033.stdout:1469: done (1 left) 2024-09-21T09:55:56.986 INFO:tasks.rados.rados.0.smithi033.stdout:1472: delete oid 31 current snap is 128 2024-09-21T09:55:56.991 INFO:tasks.rados.rados.0.smithi033.stdout:1471: finishing rollback tid 0 to smithi03385152-14 2024-09-21T09:55:56.991 INFO:tasks.rados.rados.0.smithi033.stdout:1471: finishing rollback tid 1 to smithi03385152-14 2024-09-21T09:55:56.993 INFO:tasks.rados.rados.0.smithi033.stdout:1471: finishing rollback tid 2 to smithi03385152-14 2024-09-21T09:55:56.993 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 828 (ObjNum 455 snap 120 seq_num 455) dirty exists 2024-09-21T09:55:56.995 INFO:tasks.rados.rados.0.smithi033.stdout:1471: done (1 left) 2024-09-21T09:55:56.995 INFO:tasks.rados.rados.0.smithi033.stdout:1472: done (0 left) 2024-09-21T09:55:56.995 INFO:tasks.rados.rados.0.smithi033.stdout:1473: snap_remove snap 127 2024-09-21T09:55:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:56 smithi033 ceph-mon[98546]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:55:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:56 smithi033 ceph-mon[98546]: pgmap v123: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.4 MiB/s wr, 37 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-21T09:55:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:56 smithi033 ceph-mon[96568]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:55:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:56 smithi033 ceph-mon[96568]: pgmap v123: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.4 MiB/s wr, 37 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-21T09:55:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:56 smithi105 ceph-mon[86649]: osdmap e329: 8 total, 8 up, 8 in 2024-09-21T09:55:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:56 smithi105 ceph-mon[86649]: pgmap v123: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.4 MiB/s wr, 37 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-21T09:55:57.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:55:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:55:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:55:57.989 INFO:tasks.rados.rados.0.smithi033.stdout:1473: done (0 left) 2024-09-21T09:55:57.989 INFO:tasks.rados.rados.0.smithi033.stdout:1474: read oid 12 snap 124 2024-09-21T09:55:57.989 INFO:tasks.rados.rados.0.smithi033.stdout:1474: expect (ObjNum 411 snap 110 seq_num 411) 2024-09-21T09:55:57.989 INFO:tasks.rados.rados.0.smithi033.stdout:1475: snap_remove snap 126 2024-09-21T09:55:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:57 smithi033 ceph-mon[98546]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 72/303 objects degraded (23.762%), 16 pgs degraded) 2024-09-21T09:55:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:57 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T09:55:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:57 smithi033 ceph-mon[98546]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:55:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:57 smithi033 ceph-mon[96568]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 72/303 objects degraded (23.762%), 16 pgs degraded) 2024-09-21T09:55:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:57 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T09:55:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:57 smithi033 ceph-mon[96568]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:55:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:57 smithi105 ceph-mon[86649]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 72/303 objects degraded (23.762%), 16 pgs degraded) 2024-09-21T09:55:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:57 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T09:55:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:57 smithi105 ceph-mon[86649]: osdmap e330: 8 total, 8 up, 8 in 2024-09-21T09:55:58.453 INFO:tasks.rados.rados.0.smithi033.stdout:1475: done (1 left) 2024-09-21T09:55:58.454 INFO:tasks.rados.rados.0.smithi033.stdout:1476: read oid 11 snap -1 2024-09-21T09:55:58.454 INFO:tasks.rados.rados.0.smithi033.stdout:1476: expect deleted 2024-09-21T09:55:58.454 INFO:tasks.rados.rados.0.smithi033.stdout:1477: read oid 46 snap -1 2024-09-21T09:55:58.454 INFO:tasks.rados.rados.0.smithi033.stdout:1477: expect (ObjNum 453 snap 120 seq_num 453) 2024-09-21T09:55:58.454 INFO:tasks.rados.rados.0.smithi033.stdout:1478: read oid 5 snap -1 2024-09-21T09:55:58.454 INFO:tasks.rados.rados.0.smithi033.stdout:1478: expect (ObjNum 483 snap 127 seq_num 483) 2024-09-21T09:55:58.454 INFO:tasks.rados.rados.0.smithi033.stdout:1479: copy_from oid 42 from oid 37 current snap is 128 2024-09-21T09:55:58.454 INFO:tasks.rados.rados.0.smithi033.stdout:1474: expect (ObjNum 411 snap 110 seq_num 411) 2024-09-21T09:55:58.520 INFO:tasks.rados.rados.0.smithi033.stdout:1477: expect (ObjNum 453 snap 120 seq_num 453) 2024-09-21T09:55:58.572 INFO:tasks.rados.rados.0.smithi033.stdout:1479: finishing copy_from racing read to smithi03385152-42 2024-09-21T09:55:58.572 INFO:tasks.rados.rados.0.smithi033.stdout:1479: finishing copy_from to smithi03385152-42 2024-09-21T09:55:58.572 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 707 (ObjNum 427 snap 113 seq_num 427) dirty exists 2024-09-21T09:55:58.572 INFO:tasks.rados.rados.0.smithi033.stdout:1478: expect (ObjNum 483 snap 127 seq_num 483) 2024-09-21T09:55:58.830 INFO:tasks.rados.rados.0.smithi033.stdout:1474: done (4 left) 2024-09-21T09:55:58.831 INFO:tasks.rados.rados.0.smithi033.stdout:1476: done (3 left) 2024-09-21T09:55:58.831 INFO:tasks.rados.rados.0.smithi033.stdout:1477: done (2 left) 2024-09-21T09:55:58.831 INFO:tasks.rados.rados.0.smithi033.stdout:1478: done (1 left) 2024-09-21T09:55:58.831 INFO:tasks.rados.rados.0.smithi033.stdout:1479: done (0 left) 2024-09-21T09:55:58.831 INFO:tasks.rados.rados.0.smithi033.stdout:1480: delete oid 50 current snap is 128 2024-09-21T09:55:58.832 INFO:tasks.rados.rados.0.smithi033.stdout:1480: done (0 left) 2024-09-21T09:55:58.832 INFO:tasks.rados.rados.0.smithi033.stdout:1481: read oid 4 snap 124 2024-09-21T09:55:58.832 INFO:tasks.rados.rados.0.smithi033.stdout:1481: expect (ObjNum 441 snap 117 seq_num 441) 2024-09-21T09:55:58.833 INFO:tasks.rados.rados.0.smithi033.stdout:1482: read oid 31 snap -1 2024-09-21T09:55:58.833 INFO:tasks.rados.rados.0.smithi033.stdout:1482: expect deleted 2024-09-21T09:55:58.833 INFO:tasks.rados.rados.0.smithi033.stdout:1483: delete oid 36 current snap is 128 2024-09-21T09:55:58.835 INFO:tasks.rados.rados.0.smithi033.stdout:1482: done (2 left) 2024-09-21T09:55:58.835 INFO:tasks.rados.rados.0.smithi033.stdout:1483: done (1 left) 2024-09-21T09:55:58.835 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 128 2024-09-21T09:55:58.835 INFO:tasks.rados.rados.0.smithi033.stdout:1484: seq_num 492 ranges {6184960=647168,6832128=401408} 2024-09-21T09:55:58.873 INFO:tasks.rados.rados.0.smithi033.stdout:1484: writing smithi03385152-14 from 6184960 to 6832128 tid 1 2024-09-21T09:55:58.876 INFO:tasks.rados.rados.0.smithi033.stdout:1484: writing smithi03385152-14 from 6832128 to 7233536 tid 2 2024-09-21T09:55:58.876 INFO:tasks.rados.rados.0.smithi033.stdout:1485: read oid 39 snap 124 2024-09-21T09:55:58.876 INFO:tasks.rados.rados.0.smithi033.stdout:1485: expect deleted 2024-09-21T09:55:58.876 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 128 2024-09-21T09:55:58.876 INFO:tasks.rados.rados.0.smithi033.stdout:1486: seq_num 493 ranges {5054464=663552,5718016=401408} 2024-09-21T09:55:58.907 INFO:tasks.rados.rados.0.smithi033.stdout:1486: writing smithi03385152-21 from 5054464 to 5718016 tid 1 2024-09-21T09:55:58.909 INFO:tasks.rados.rados.0.smithi033.stdout:1486: writing smithi03385152-21 from 5718016 to 6119424 tid 2 2024-09-21T09:55:58.909 INFO:tasks.rados.rados.0.smithi033.stdout:1487: delete oid 41 current snap is 128 2024-09-21T09:55:58.909 INFO:tasks.rados.rados.0.smithi033.stdout:1481: expect (ObjNum 441 snap 117 seq_num 441) 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1484: finishing write tid 1 to smithi03385152-14 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1484: finishing write tid 2 to smithi03385152-14 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1481: done (4 left) 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1485: done (3 left) 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1487: done (2 left) 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1488: setattr oid 22 current snap is 128 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1484: finishing write tid 3 to smithi03385152-14 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1484: finishing write tid 4 to smithi03385152-14 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 835 (ObjNum 492 snap 128 seq_num 492) dirty exists 2024-09-21T09:55:59.108 INFO:tasks.rados.rados.0.smithi033.stdout:1484: left oid 14 (ObjNum 492 snap 128 seq_num 492) 2024-09-21T09:55:59.109 INFO:tasks.rados.rados.0.smithi033.stdout:1486: finishing write tid 1 to smithi03385152-21 2024-09-21T09:55:59.109 INFO:tasks.rados.rados.0.smithi033.stdout:1486: finishing write tid 2 to smithi03385152-21 2024-09-21T09:55:59.109 INFO:tasks.rados.rados.0.smithi033.stdout:1486: finishing write tid 3 to smithi03385152-21 2024-09-21T09:55:59.109 INFO:tasks.rados.rados.0.smithi033.stdout:1486: finishing write tid 4 to smithi03385152-21 2024-09-21T09:55:59.109 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 694 (ObjNum 493 snap 128 seq_num 493) dirty exists 2024-09-21T09:55:59.109 INFO:tasks.rados.rados.0.smithi033.stdout:1486: left oid 21 (ObjNum 493 snap 128 seq_num 493) 2024-09-21T09:55:59.109 INFO:tasks.rados.rados.0.smithi033.stdout:1484: done (2 left) 2024-09-21T09:55:59.110 INFO:tasks.rados.rados.0.smithi033.stdout:1486: done (1 left) 2024-09-21T09:55:59.110 INFO:tasks.rados.rados.0.smithi033.stdout:1489: snap_remove snap 124 2024-09-21T09:55:59.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:58 smithi033 ceph-mon[98546]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:55:59.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:58 smithi033 ceph-mon[98546]: pgmap v126: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.4 MiB/s wr, 37 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-21T09:55:59.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:55:58 smithi033 ceph-mon[98546]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:55:59.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:58 smithi033 ceph-mon[96568]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:55:59.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:58 smithi033 ceph-mon[96568]: pgmap v126: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.4 MiB/s wr, 37 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-21T09:55:59.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:55:58 smithi033 ceph-mon[96568]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:55:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:58 smithi105 ceph-mon[86649]: osdmap e331: 8 total, 8 up, 8 in 2024-09-21T09:55:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:58 smithi105 ceph-mon[86649]: pgmap v126: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 2.4 MiB/s wr, 37 op/s; 23 MiB/s, 5 objects/s recovering 2024-09-21T09:55:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:55:58 smithi105 ceph-mon[86649]: osdmap e332: 8 total, 8 up, 8 in 2024-09-21T09:55:59.995 INFO:tasks.rados.rados.0.smithi033.stdout:1489: done (1 left) 2024-09-21T09:55:59.995 INFO:tasks.rados.rados.0.smithi033.stdout:1490: setattr oid 50 current snap is 128 2024-09-21T09:55:59.995 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 837 (ObjNum 419 snap 111 seq_num 419) dirty exists 2024-09-21T09:55:59.998 INFO:tasks.rados.rados.0.smithi033.stdout:1488: done (1 left) 2024-09-21T09:55:59.998 INFO:tasks.rados.rados.0.smithi033.stdout:1491: rollback oid 32 current snap is 128 2024-09-21T09:55:59.998 INFO:tasks.rados.rados.0.smithi033.stdout:1491: done (1 left) 2024-09-21T09:55:59.998 INFO:tasks.rados.rados.0.smithi033.stdout:1492: read oid 37 snap -1 2024-09-21T09:55:59.998 INFO:tasks.rados.rados.0.smithi033.stdout:1492: expect (ObjNum 427 snap 113 seq_num 427) 2024-09-21T09:55:59.998 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 128 2024-09-21T09:55:59.998 INFO:tasks.rados.rados.0.smithi033.stdout:1493: seq_num 495 ranges {376832=696320,1073152=401408} 2024-09-21T09:56:00.006 INFO:tasks.rados.rados.0.smithi033.stdout:1493: writing smithi03385152-6 from 376832 to 1073152 tid 1 2024-09-21T09:56:00.009 INFO:tasks.rados.rados.0.smithi033.stdout:1493: writing smithi03385152-6 from 1073152 to 1474560 tid 2 2024-09-21T09:56:00.009 INFO:tasks.rados.rados.0.smithi033.stdout:1494: read oid 4 snap -1 2024-09-21T09:56:00.009 INFO:tasks.rados.rados.0.smithi033.stdout:1494: expect (ObjNum 484 snap 127 seq_num 484) 2024-09-21T09:56:00.010 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 128 2024-09-21T09:56:00.010 INFO:tasks.rados.rados.0.smithi033.stdout:1495: seq_num 496 ranges {0=712704,712704=401408} 2024-09-21T09:56:00.018 INFO:tasks.rados.rados.0.smithi033.stdout:1495: writing smithi03385152-20 from 0 to 712704 tid 1 2024-09-21T09:56:00.018 INFO:tasks.rados.rados.0.smithi033.stdout:1495: writing smithi03385152-20 from 712704 to 1114112 tid 2 2024-09-21T09:56:00.018 INFO:tasks.rados.rados.0.smithi033.stdout:1496: rollback oid 32 current snap is 128 2024-09-21T09:56:00.019 INFO:tasks.rados.rados.0.smithi033.stdout:1496: done (5 left) 2024-09-21T09:56:00.019 INFO:tasks.rados.rados.0.smithi033.stdout:1497: rollback oid 28 current snap is 128 2024-09-21T09:56:00.019 INFO:tasks.rados.rados.0.smithi033.stdout:1497: done (5 left) 2024-09-21T09:56:00.019 INFO:tasks.rados.rados.0.smithi033.stdout:1498: delete oid 35 current snap is 128 2024-09-21T09:56:00.019 INFO:tasks.rados.rados.0.smithi033.stdout:1493: finishing write tid 1 to smithi03385152-6 2024-09-21T09:56:00.019 INFO:tasks.rados.rados.0.smithi033.stdout:1493: finishing write tid 2 to smithi03385152-6 2024-09-21T09:56:00.020 INFO:tasks.rados.rados.0.smithi033.stdout:1493: finishing write tid 3 to smithi03385152-6 2024-09-21T09:56:00.020 INFO:tasks.rados.rados.0.smithi033.stdout:1493: finishing write tid 4 to smithi03385152-6 2024-09-21T09:56:00.020 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 492 (ObjNum 495 snap 128 seq_num 495) dirty exists 2024-09-21T09:56:00.020 INFO:tasks.rados.rados.0.smithi033.stdout:1493: left oid 6 (ObjNum 495 snap 128 seq_num 495) 2024-09-21T09:56:00.021 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 641 (ObjNum 33 snap 0 seq_num 1497751971) dirty exists 2024-09-21T09:56:00.023 INFO:tasks.rados.rados.0.smithi033.stdout:1495: finishing write tid 1 to smithi03385152-20 2024-09-21T09:56:00.024 INFO:tasks.rados.rados.0.smithi033.stdout:1495: finishing write tid 2 to smithi03385152-20 2024-09-21T09:56:00.024 INFO:tasks.rados.rados.0.smithi033.stdout:1495: finishing write tid 3 to smithi03385152-20 2024-09-21T09:56:00.026 INFO:tasks.rados.rados.0.smithi033.stdout:1492: expect (ObjNum 427 snap 113 seq_num 427) 2024-09-21T09:56:00.093 INFO:tasks.rados.rados.0.smithi033.stdout:1495: finishing write tid 4 to smithi03385152-20 2024-09-21T09:56:00.093 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 697 (ObjNum 496 snap 128 seq_num 496) dirty exists 2024-09-21T09:56:00.093 INFO:tasks.rados.rados.0.smithi033.stdout:1495: left oid 20 (ObjNum 496 snap 128 seq_num 496) 2024-09-21T09:56:00.093 INFO:tasks.rados.rados.0.smithi033.stdout:1494: expect (ObjNum 484 snap 127 seq_num 484) 2024-09-21T09:56:00.358 INFO:tasks.rados.rados.0.smithi033.stdout:1490: done (5 left) 2024-09-21T09:56:00.358 INFO:tasks.rados.rados.0.smithi033.stdout:1492: done (4 left) 2024-09-21T09:56:00.358 INFO:tasks.rados.rados.0.smithi033.stdout:1493: done (3 left) 2024-09-21T09:56:00.358 INFO:tasks.rados.rados.0.smithi033.stdout:1494: done (2 left) 2024-09-21T09:56:00.358 INFO:tasks.rados.rados.0.smithi033.stdout:1495: done (1 left) 2024-09-21T09:56:00.359 INFO:tasks.rados.rados.0.smithi033.stdout:1498: done (0 left) 2024-09-21T09:56:00.359 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 128 2024-09-21T09:56:00.359 INFO:tasks.rados.rados.0.smithi033.stdout:1499: seq_num 498 ranges {12713984=745472,13459456=401408} 2024-09-21T09:56:00.434 INFO:tasks.rados.rados.0.smithi033.stdout:1499: writing smithi03385152-24 from 12713984 to 13459456 tid 1 2024-09-21T09:56:00.434 INFO:tasks.rados.rados.0.smithi033.stdout:1499: writing smithi03385152-24 from 13459456 to 13860864 tid 2 2024-09-21T09:56:00.434 INFO:tasks.rados.rados.0.smithi033.stdout:1500: read oid 5 snap -1 2024-09-21T09:56:00.435 INFO:tasks.rados.rados.0.smithi033.stdout:1500: expect (ObjNum 483 snap 127 seq_num 483) 2024-09-21T09:56:00.435 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 128 2024-09-21T09:56:00.435 INFO:tasks.rados.rados.0.smithi033.stdout:1501: seq_num 499 ranges {5701632=761856,6463488=401408} 2024-09-21T09:56:00.470 INFO:tasks.rados.rados.0.smithi033.stdout:1501: writing smithi03385152-38 from 5701632 to 6463488 tid 1 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1501: writing smithi03385152-38 from 6463488 to 6864896 tid 2 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1502: read oid 7 snap -1 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1502: expect (ObjNum 474 snap 125 seq_num 474) 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1503: rollback oid 25 current snap is 128 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1503: done (4 left) 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1504: snap_create 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1499: finishing write tid 1 to smithi03385152-24 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1499: finishing write tid 2 to smithi03385152-24 2024-09-21T09:56:00.472 INFO:tasks.rados.rados.0.smithi033.stdout:1499: finishing write tid 3 to smithi03385152-24 2024-09-21T09:56:00.473 INFO:tasks.rados.rados.0.smithi033.stdout:1499: finishing write tid 4 to smithi03385152-24 2024-09-21T09:56:00.473 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 655 (ObjNum 498 snap 128 seq_num 498) dirty exists 2024-09-21T09:56:00.473 INFO:tasks.rados.rados.0.smithi033.stdout:1499: left oid 24 (ObjNum 498 snap 128 seq_num 498) 2024-09-21T09:56:00.473 INFO:tasks.rados.rados.0.smithi033.stdout:1500: expect (ObjNum 483 snap 127 seq_num 483) 2024-09-21T09:56:00.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:00.733 INFO:tasks.rados.rados.0.smithi033.stdout:1501: finishing write tid 1 to smithi03385152-38 2024-09-21T09:56:00.733 INFO:tasks.rados.rados.0.smithi033.stdout:1501: finishing write tid 2 to smithi03385152-38 2024-09-21T09:56:00.733 INFO:tasks.rados.rados.0.smithi033.stdout:1501: finishing write tid 3 to smithi03385152-38 2024-09-21T09:56:00.733 INFO:tasks.rados.rados.0.smithi033.stdout:1502: expect (ObjNum 474 snap 125 seq_num 474) 2024-09-21T09:56:00.753 INFO:tasks.rados.rados.0.smithi033.stdout:1501: finishing write tid 4 to smithi03385152-38 2024-09-21T09:56:00.753 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 717 (ObjNum 499 snap 128 seq_num 499) dirty exists 2024-09-21T09:56:00.753 INFO:tasks.rados.rados.0.smithi033.stdout:1501: left oid 38 (ObjNum 499 snap 128 seq_num 499) 2024-09-21T09:56:01.000 INFO:tasks.rados.rados.0.smithi033.stdout:1499: done (4 left) 2024-09-21T09:56:01.000 INFO:tasks.rados.rados.0.smithi033.stdout:1500: done (3 left) 2024-09-21T09:56:01.000 INFO:tasks.rados.rados.0.smithi033.stdout:1501: done (2 left) 2024-09-21T09:56:01.000 INFO:tasks.rados.rados.0.smithi033.stdout:1502: done (1 left) 2024-09-21T09:56:01.000 INFO:tasks.rados.rados.0.smithi033.stdout:1504: done (0 left) 2024-09-21T09:56:01.001 INFO:tasks.rados.rados.0.smithi033.stdout:1505: rollback oid 1 current snap is 129 2024-09-21T09:56:01.001 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 128 2024-09-21T09:56:01.001 INFO:tasks.rados.rados.0.smithi033.stdout:1506: read oid 32 snap -1 2024-09-21T09:56:01.001 INFO:tasks.rados.rados.0.smithi033.stdout:1506: expect (ObjNum 466 snap 124 seq_num 466) 2024-09-21T09:56:01.001 INFO:tasks.rados.rados.0.smithi033.stdout:1507: snap_create 2024-09-21T09:56:01.005 INFO:tasks.rados.rados.0.smithi033.stdout:1505: finishing rollback tid 0 to smithi03385152-1 2024-09-21T09:56:01.005 INFO:tasks.rados.rados.0.smithi033.stdout:1505: finishing rollback tid 1 to smithi03385152-1 2024-09-21T09:56:01.005 INFO:tasks.rados.rados.0.smithi033.stdout:1505: finishing rollback tid 2 to smithi03385152-1 2024-09-21T09:56:01.006 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 652 (ObjNum 404 snap 110 seq_num 404) dirty exists 2024-09-21T09:56:01.008 INFO:tasks.rados.rados.0.smithi033.stdout:1506: expect (ObjNum 466 snap 124 seq_num 466) 2024-09-21T09:56:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:01 smithi105 ceph-mon[86649]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:56:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:01 smithi105 ceph-mon[86649]: pgmap v129: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:01.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:00 smithi033 ceph-mon[96568]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:56:01.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:00 smithi033 ceph-mon[96568]: pgmap v129: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:01.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:01.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:01.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:00 smithi033 ceph-mon[98546]: osdmap e333: 8 total, 8 up, 8 in 2024-09-21T09:56:01.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:00 smithi033 ceph-mon[98546]: pgmap v129: 121 pgs: 121 active+clean; 153 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:01.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:01.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:56:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:56:01] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.51.0" 2024-09-21T09:56:02.001 INFO:tasks.rados.rados.0.smithi033.stdout:1505: done (2 left) 2024-09-21T09:56:02.002 INFO:tasks.rados.rados.0.smithi033.stdout:1506: done (1 left) 2024-09-21T09:56:02.002 INFO:tasks.rados.rados.0.smithi033.stdout:1507: done (0 left) 2024-09-21T09:56:02.002 INFO:tasks.rados.rados.0.smithi033.stdout:1508: snap_create 2024-09-21T09:56:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:02 smithi105 ceph-mon[86649]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:56:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:02 smithi033 ceph-mon[96568]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:56:02.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:02 smithi033 ceph-mon[98546]: osdmap e334: 8 total, 8 up, 8 in 2024-09-21T09:56:03.005 INFO:tasks.rados.rados.0.smithi033.stdout:1508: done (0 left) 2024-09-21T09:56:03.005 INFO:tasks.rados.rados.0.smithi033.stdout:1509: copy_from oid 10 from oid 15 current snap is 131 2024-09-21T09:56:03.005 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 131 2024-09-21T09:56:03.005 INFO:tasks.rados.rados.0.smithi033.stdout:1510: seq_num 500 ranges {0=778240,778240=401408} 2024-09-21T09:56:03.010 INFO:tasks.rados.rados.0.smithi033.stdout:1510: writing smithi03385152-35 from 0 to 778240 tid 1 2024-09-21T09:56:03.012 INFO:tasks.rados.rados.0.smithi033.stdout:1510: writing smithi03385152-35 from 778240 to 1179648 tid 2 2024-09-21T09:56:03.012 INFO:tasks.rados.rados.0.smithi033.stdout:1511: copy_from oid 6 from oid 14 current snap is 131 2024-09-21T09:56:03.012 INFO:tasks.rados.rados.0.smithi033.stdout:1512: snap_create 2024-09-21T09:56:03.022 INFO:tasks.rados.rados.0.smithi033.stdout:1510: finishing write tid 1 to smithi03385152-35 2024-09-21T09:56:03.022 INFO:tasks.rados.rados.0.smithi033.stdout:1510: finishing write tid 2 to smithi03385152-35 2024-09-21T09:56:03.022 INFO:tasks.rados.rados.0.smithi033.stdout:1510: finishing write tid 3 to smithi03385152-35 2024-09-21T09:56:03.028 INFO:tasks.rados.rados.0.smithi033.stdout:1510: finishing write tid 4 to smithi03385152-35 2024-09-21T09:56:03.028 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 964 (ObjNum 500 snap 131 seq_num 500) dirty exists 2024-09-21T09:56:03.028 INFO:tasks.rados.rados.0.smithi033.stdout:1510: left oid 35 (ObjNum 500 snap 131 seq_num 500) 2024-09-21T09:56:03.031 INFO:tasks.rados.rados.0.smithi033.stdout:1509: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:56:03.031 INFO:tasks.rados.rados.0.smithi033.stdout:1509: finishing copy_from to smithi03385152-10 2024-09-21T09:56:03.031 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 649 (ObjNum 486 snap 127 seq_num 486) dirty exists 2024-09-21T09:56:03.043 INFO:tasks.rados.rados.0.smithi033.stdout:1511: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:56:03.048 INFO:tasks.rados.rados.0.smithi033.stdout:1511: finishing copy_from to smithi03385152-6 2024-09-21T09:56:03.048 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 835 (ObjNum 492 snap 128 seq_num 492) dirty exists 2024-09-21T09:56:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:03 smithi105 ceph-mon[86649]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:56:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:03 smithi105 ceph-mon[86649]: pgmap v132: 121 pgs: 121 active+clean; 125 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 29 op/s 2024-09-21T09:56:03.450 INFO:tasks.rados.rados.0.smithi033.stdout:1509: done (3 left) 2024-09-21T09:56:03.450 INFO:tasks.rados.rados.0.smithi033.stdout:1510: done (2 left) 2024-09-21T09:56:03.451 INFO:tasks.rados.rados.0.smithi033.stdout:1511: done (1 left) 2024-09-21T09:56:03.451 INFO:tasks.rados.rados.0.smithi033.stdout:1512: done (0 left) 2024-09-21T09:56:03.451 INFO:tasks.rados.rados.0.smithi033.stdout:1513: delete oid 36 current snap is 132 2024-09-21T09:56:03.455 INFO:tasks.rados.rados.0.smithi033.stdout:1513: done (0 left) 2024-09-21T09:56:03.455 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 132 2024-09-21T09:56:03.455 INFO:tasks.rados.rados.0.smithi033.stdout:1514: seq_num 502 ranges {3678208=409600,4087808=417792,4505600=385024} 2024-09-21T09:56:03.477 INFO:tasks.rados.rados.0.smithi033.stdout:1514: writing smithi03385152-4 from 3678208 to 4087808 tid 1 2024-09-21T09:56:03.480 INFO:tasks.rados.rados.0.smithi033.stdout:1514: writing smithi03385152-4 from 4087808 to 4505600 tid 2 2024-09-21T09:56:03.482 INFO:tasks.rados.rados.0.smithi033.stdout:1514: writing smithi03385152-4 from 4505600 to 4890624 tid 3 2024-09-21T09:56:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:1515: rollback oid 39 current snap is 132 2024-09-21T09:56:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 39 to 131 2024-09-21T09:56:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:1516: read oid 48 snap 128 2024-09-21T09:56:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:1516: expect deleted 2024-09-21T09:56:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:1517: rollback oid 36 current snap is 132 2024-09-21T09:56:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 131 2024-09-21T09:56:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:1518: rmattr oid 5 current snap is 132 2024-09-21T09:56:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:1519: setattr oid 14 current snap is 132 2024-09-21T09:56:03.484 INFO:tasks.rados.rados.0.smithi033.stdout:1517: finishing rollback tid 1 to smithi03385152-36 2024-09-21T09:56:03.484 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 0 (ObjNum 1265 snap 0 seq_num 1188656392) dirty dne 2024-09-21T09:56:03.484 INFO:tasks.rados.rados.0.smithi033.stdout:1516: done (5 left) 2024-09-21T09:56:03.484 INFO:tasks.rados.rados.0.smithi033.stdout:1517: done (4 left) 2024-09-21T09:56:03.484 INFO:tasks.rados.rados.0.smithi033.stdout:1520: read oid 12 snap 128 2024-09-21T09:56:03.484 INFO:tasks.rados.rados.0.smithi033.stdout:1520: expect (ObjNum 480 snap 125 seq_num 480) 2024-09-21T09:56:03.484 INFO:tasks.rados.rados.0.smithi033.stdout:1521: setattr oid 41 current snap is 132 2024-09-21T09:56:03.485 INFO:tasks.rados.rados.0.smithi033.stdout:1522: snap_remove snap 130 2024-09-21T09:56:03.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:03 smithi033 ceph-mon[96568]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:56:03.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:03 smithi033 ceph-mon[96568]: pgmap v132: 121 pgs: 121 active+clean; 125 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 29 op/s 2024-09-21T09:56:03.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:03 smithi033 ceph-mon[98546]: osdmap e335: 8 total, 8 up, 8 in 2024-09-21T09:56:03.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:03 smithi033 ceph-mon[98546]: pgmap v132: 121 pgs: 121 active+clean; 125 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 29 op/s 2024-09-21T09:56:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:04 smithi105 ceph-mon[86649]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:56:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:04 smithi105 ceph-mon[86649]: mgrmap e44: y(active, since 92s), standbys: x 2024-09-21T09:56:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:04 smithi105 ceph-mon[86649]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:56:04.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:1522: done (6 left) 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 132 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:1514: finishing write tid 1 to smithi03385152-4 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:1514: finishing write tid 2 to smithi03385152-4 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 718 (ObjNum 33 snap 0 seq_num 72668536) dirty exists 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:1514: finishing write tid 3 to smithi03385152-4 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 843 (ObjNum 492 snap 128 seq_num 492) dirty exists 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:1514: finishing write tid 4 to smithi03385152-4 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:1515: finishing rollback tid 1 to smithi03385152-39 2024-09-21T09:56:04.454 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 0 (ObjNum 1297 snap 0 seq_num 0) dirty dne 2024-09-21T09:56:04.455 INFO:tasks.rados.rados.0.smithi033.stdout:1514: finishing write tid 5 to smithi03385152-4 2024-09-21T09:56:04.455 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 699 (ObjNum 502 snap 132 seq_num 502) dirty exists 2024-09-21T09:56:04.455 INFO:tasks.rados.rados.0.smithi033.stdout:1514: left oid 4 (ObjNum 502 snap 132 seq_num 502) 2024-09-21T09:56:04.455 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 200 (ObjNum 483 snap 127 seq_num 483) dirty exists 2024-09-21T09:56:04.455 INFO:tasks.rados.rados.0.smithi033.stdout:1520: expect (ObjNum 480 snap 125 seq_num 480) 2024-09-21T09:56:04.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:04 smithi033 ceph-mon[98546]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:56:04.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:04 smithi033 ceph-mon[98546]: mgrmap e44: y(active, since 92s), standbys: x 2024-09-21T09:56:04.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:04 smithi033 ceph-mon[98546]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:56:04.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:04 smithi033 ceph-mon[96568]: osdmap e336: 8 total, 8 up, 8 in 2024-09-21T09:56:04.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:04 smithi033 ceph-mon[96568]: mgrmap e44: y(active, since 92s), standbys: x 2024-09-21T09:56:04.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:04 smithi033 ceph-mon[96568]: osdmap e337: 8 total, 8 up, 8 in 2024-09-21T09:56:04.568 INFO:tasks.rados.rados.0.smithi033.stdout:1523: seq_num 503 ranges {5505024=425984,5931008=704512,6635520=98304} 2024-09-21T09:56:04.600 INFO:tasks.rados.rados.0.smithi033.stdout:1523: writing smithi03385152-47 from 5505024 to 5931008 tid 1 2024-09-21T09:56:04.604 INFO:tasks.rados.rados.0.smithi033.stdout:1523: writing smithi03385152-47 from 5931008 to 6635520 tid 2 2024-09-21T09:56:04.605 INFO:tasks.rados.rados.0.smithi033.stdout:1523: writing smithi03385152-47 from 6635520 to 6733824 tid 3 2024-09-21T09:56:04.605 INFO:tasks.rados.rados.0.smithi033.stdout:1514: done (6 left) 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:1515: done (5 left) 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:1518: done (4 left) 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:1519: done (3 left) 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:1520: done (2 left) 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:1521: done (1 left) 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:1524: rollback oid 16 current snap is 132 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 16 to 128 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:1523: finishing write tid 1 to smithi03385152-47 2024-09-21T09:56:04.606 INFO:tasks.rados.rados.0.smithi033.stdout:1525: rollback oid 24 current snap is 132 2024-09-21T09:56:04.607 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 131 2024-09-21T09:56:04.607 INFO:tasks.rados.rados.0.smithi033.stdout:1526: read oid 33 snap -1 2024-09-21T09:56:04.607 INFO:tasks.rados.rados.0.smithi033.stdout:1526: expect deleted 2024-09-21T09:56:04.607 INFO:tasks.rados.rados.0.smithi033.stdout:1527: read oid 48 snap -1 2024-09-21T09:56:04.607 INFO:tasks.rados.rados.0.smithi033.stdout:1527: expect deleted 2024-09-21T09:56:04.607 INFO:tasks.rados.rados.0.smithi033.stdout:1528: setattr oid 9 current snap is 132 2024-09-21T09:56:04.607 INFO:tasks.rados.rados.0.smithi033.stdout:1524: finishing rollback tid 1 to smithi03385152-16 2024-09-21T09:56:04.607 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:56:04.608 INFO:tasks.rados.rados.0.smithi033.stdout:1523: finishing write tid 2 to smithi03385152-47 2024-09-21T09:56:04.608 INFO:tasks.rados.rados.0.smithi033.stdout:1523: finishing write tid 3 to smithi03385152-47 2024-09-21T09:56:04.608 INFO:tasks.rados.rados.0.smithi033.stdout:1525: finishing rollback tid 0 to smithi03385152-24 2024-09-21T09:56:04.608 INFO:tasks.rados.rados.0.smithi033.stdout:1525: finishing rollback tid 1 to smithi03385152-24 2024-09-21T09:56:04.608 INFO:tasks.rados.rados.0.smithi033.stdout:1524: done (5 left) 2024-09-21T09:56:04.608 INFO:tasks.rados.rados.0.smithi033.stdout:1526: done (4 left) 2024-09-21T09:56:04.608 INFO:tasks.rados.rados.0.smithi033.stdout:1527: done (3 left) 2024-09-21T09:56:04.608 INFO:tasks.rados.rados.0.smithi033.stdout:1529: snap_remove snap 128 2024-09-21T09:56:05.458 INFO:tasks.rados.rados.0.smithi033.stdout:1529: done (3 left) 2024-09-21T09:56:05.458 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 132 2024-09-21T09:56:05.458 INFO:tasks.rados.rados.0.smithi033.stdout:1530: seq_num 504 ranges {704512=450560,1155072=589824,1744896=212992} 2024-09-21T09:56:05.465 INFO:tasks.rados.rados.0.smithi033.stdout:1530: writing smithi03385152-29 from 704512 to 1155072 tid 1 2024-09-21T09:56:05.469 INFO:tasks.rados.rados.0.smithi033.stdout:1530: writing smithi03385152-29 from 1155072 to 1744896 tid 2 2024-09-21T09:56:05.471 INFO:tasks.rados.rados.0.smithi033.stdout:1530: writing smithi03385152-29 from 1744896 to 1957888 tid 3 2024-09-21T09:56:05.471 INFO:tasks.rados.rados.0.smithi033.stdout:1531: snap_create 2024-09-21T09:56:05.471 INFO:tasks.rados.rados.0.smithi033.stdout:1523: finishing write tid 4 to smithi03385152-47 2024-09-21T09:56:05.472 INFO:tasks.rados.rados.0.smithi033.stdout:1525: finishing rollback tid 2 to smithi03385152-24 2024-09-21T09:56:05.472 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 658 (ObjNum 498 snap 128 seq_num 498) dirty exists 2024-09-21T09:56:05.472 INFO:tasks.rados.rados.0.smithi033.stdout:1523: finishing write tid 5 to smithi03385152-47 2024-09-21T09:56:05.472 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 649 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:56:05.472 INFO:tasks.rados.rados.0.smithi033.stdout:1523: left oid 47 (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:56:05.472 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 719 (ObjNum 479 snap 125 seq_num 479) dirty exists 2024-09-21T09:56:05.472 INFO:tasks.rados.rados.0.smithi033.stdout:1530: finishing write tid 1 to smithi03385152-29 2024-09-21T09:56:05.474 INFO:tasks.rados.rados.0.smithi033.stdout:1530: finishing write tid 2 to smithi03385152-29 2024-09-21T09:56:05.474 INFO:tasks.rados.rados.0.smithi033.stdout:1530: finishing write tid 3 to smithi03385152-29 2024-09-21T09:56:05.474 INFO:tasks.rados.rados.0.smithi033.stdout:1530: finishing write tid 4 to smithi03385152-29 2024-09-21T09:56:05.475 INFO:tasks.rados.rados.0.smithi033.stdout:1530: finishing write tid 5 to smithi03385152-29 2024-09-21T09:56:05.475 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 970 (ObjNum 504 snap 132 seq_num 504) dirty exists 2024-09-21T09:56:05.475 INFO:tasks.rados.rados.0.smithi033.stdout:1530: left oid 29 (ObjNum 504 snap 132 seq_num 504) 2024-09-21T09:56:05.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:05 smithi105 ceph-mon[86649]: pgmap v135: 121 pgs: 121 active+clean; 125 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 29 op/s 2024-09-21T09:56:05.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:05 smithi105 ceph-mon[86649]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:56:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:05 smithi033 ceph-mon[98546]: pgmap v135: 121 pgs: 121 active+clean; 125 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 29 op/s 2024-09-21T09:56:05.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:05 smithi033 ceph-mon[98546]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:56:05.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:05 smithi033 ceph-mon[96568]: pgmap v135: 121 pgs: 121 active+clean; 125 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 2.1 MiB/s wr, 29 op/s 2024-09-21T09:56:05.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:05 smithi033 ceph-mon[96568]: osdmap e338: 8 total, 8 up, 8 in 2024-09-21T09:56:06.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:06.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:06.465 INFO:tasks.rados.rados.0.smithi033.stdout:1523: done (4 left) 2024-09-21T09:56:06.465 INFO:tasks.rados.rados.0.smithi033.stdout:1525: done (3 left) 2024-09-21T09:56:06.465 INFO:tasks.rados.rados.0.smithi033.stdout:1528: done (2 left) 2024-09-21T09:56:06.465 INFO:tasks.rados.rados.0.smithi033.stdout:1530: done (1 left) 2024-09-21T09:56:06.465 INFO:tasks.rados.rados.0.smithi033.stdout:1531: done (0 left) 2024-09-21T09:56:06.465 INFO:tasks.rados.rados.0.smithi033.stdout:1532: read oid 27 snap -1 2024-09-21T09:56:06.465 INFO:tasks.rados.rados.0.smithi033.stdout:1532: expect deleted 2024-09-21T09:56:06.466 INFO:tasks.rados.rados.0.smithi033.stdout:1533: rmattr oid 22 current snap is 133 2024-09-21T09:56:06.466 INFO:tasks.rados.rados.0.smithi033.stdout:1534: read oid 43 snap 132 2024-09-21T09:56:06.466 INFO:tasks.rados.rados.0.smithi033.stdout:1534: expect (ObjNum 410 snap 110 seq_num 410) 2024-09-21T09:56:06.466 INFO:tasks.rados.rados.0.smithi033.stdout:1535: snap_create 2024-09-21T09:56:06.472 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 849 (ObjNum 419 snap 111 seq_num 419) dirty exists 2024-09-21T09:56:06.485 INFO:tasks.rados.rados.0.smithi033.stdout:1534: expect (ObjNum 410 snap 110 seq_num 410) 2024-09-21T09:56:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:06 smithi033 ceph-mon[98546]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:56:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:06.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:06 smithi033 ceph-mon[96568]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:56:06.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:06 smithi105 ceph-mon[86649]: osdmap e339: 8 total, 8 up, 8 in 2024-09-21T09:56:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:07.473 INFO:tasks.rados.rados.0.smithi033.stdout:1532: done (3 left) 2024-09-21T09:56:07.473 INFO:tasks.rados.rados.0.smithi033.stdout:1533: done (2 left) 2024-09-21T09:56:07.473 INFO:tasks.rados.rados.0.smithi033.stdout:1534: done (1 left) 2024-09-21T09:56:07.473 INFO:tasks.rados.rados.0.smithi033.stdout:1535: done (0 left) 2024-09-21T09:56:07.473 INFO:tasks.rados.rados.0.smithi033.stdout:1536: setattr oid 18 current snap is 134 2024-09-21T09:56:07.474 INFO:tasks.rados.rados.0.smithi033.stdout:1537: setattr oid 47 current snap is 134 2024-09-21T09:56:07.476 INFO:tasks.rados.rados.0.smithi033.stdout:1538: delete oid 41 current snap is 134 2024-09-21T09:56:07.482 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 650 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:56:07.482 INFO:tasks.rados.rados.0.smithi033.stdout:1537: done (2 left) 2024-09-21T09:56:07.482 INFO:tasks.rados.rados.0.smithi033.stdout:1538: done (1 left) 2024-09-21T09:56:07.482 INFO:tasks.rados.rados.0.smithi033.stdout:1539: snap_create 2024-09-21T09:56:07.482 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 128 (ObjNum 129 snap 0 seq_num 90931768) dirty exists 2024-09-21T09:56:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:07 smithi033 ceph-mon[98546]: pgmap v138: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-21T09:56:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:07 smithi033 ceph-mon[98546]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:56:07.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:07 smithi033 ceph-mon[96568]: pgmap v138: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-21T09:56:07.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:07 smithi033 ceph-mon[96568]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:56:07.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:07 smithi105 ceph-mon[86649]: pgmap v138: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-21T09:56:07.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:07 smithi105 ceph-mon[86649]: osdmap e340: 8 total, 8 up, 8 in 2024-09-21T09:56:08.453 INFO:tasks.rados.rados.0.smithi033.stdout:1536: done (1 left) 2024-09-21T09:56:08.453 INFO:tasks.rados.rados.0.smithi033.stdout:1539: done (0 left) 2024-09-21T09:56:08.453 INFO:tasks.rados.rados.0.smithi033.stdout:1540: copy_from oid 49 from oid 48 current snap is 135 2024-09-21T09:56:08.453 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 135 2024-09-21T09:56:08.454 INFO:tasks.rados.rados.0.smithi033.stdout:1541: seq_num 506 ranges {2203648=483328,2686976=761856,3448832=40960} 2024-09-21T09:56:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1541: writing smithi03385152-15 from 2203648 to 2686976 tid 1 2024-09-21T09:56:08.473 INFO:tasks.rados.rados.0.smithi033.stdout:1541: writing smithi03385152-15 from 2686976 to 3448832 tid 2 2024-09-21T09:56:08.473 INFO:tasks.rados.rados.0.smithi033.stdout:1541: writing smithi03385152-15 from 3448832 to 3489792 tid 3 2024-09-21T09:56:08.473 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 135 2024-09-21T09:56:08.473 INFO:tasks.rados.rados.0.smithi033.stdout:1540: finishing copy_from to smithi03385152-49 2024-09-21T09:56:08.473 INFO:tasks.rados.rados.0.smithi033.stdout:1540: got expected ENOENT (src dne) 2024-09-21T09:56:08.474 INFO:tasks.rados.rados.0.smithi033.stdout:1542: seq_num 507 ranges {6201344=499712,6701056=638976,7340032=163840} 2024-09-21T09:56:08.509 INFO:tasks.rados.rados.0.smithi033.stdout:1542: writing smithi03385152-17 from 6201344 to 6701056 tid 1 2024-09-21T09:56:08.513 INFO:tasks.rados.rados.0.smithi033.stdout:1542: writing smithi03385152-17 from 6701056 to 7340032 tid 2 2024-09-21T09:56:08.517 INFO:tasks.rados.rados.0.smithi033.stdout:1542: writing smithi03385152-17 from 7340032 to 7503872 tid 3 2024-09-21T09:56:08.517 INFO:tasks.rados.rados.0.smithi033.stdout:1543: rollback oid 28 current snap is 135 2024-09-21T09:56:08.517 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 131 2024-09-21T09:56:08.518 INFO:tasks.rados.rados.0.smithi033.stdout:1540: finishing copy_from racing read to smithi03385152-49 2024-09-21T09:56:08.518 INFO:tasks.rados.rados.0.smithi033.stdout:1541: finishing write tid 1 to smithi03385152-15 2024-09-21T09:56:08.518 INFO:tasks.rados.rados.0.smithi033.stdout:1541: finishing write tid 2 to smithi03385152-15 2024-09-21T09:56:08.518 INFO:tasks.rados.rados.0.smithi033.stdout:1541: finishing write tid 3 to smithi03385152-15 2024-09-21T09:56:08.518 INFO:tasks.rados.rados.0.smithi033.stdout:1540: done (3 left) 2024-09-21T09:56:08.518 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 135 2024-09-21T09:56:08.518 INFO:tasks.rados.rados.0.smithi033.stdout:1541: finishing write tid 4 to smithi03385152-15 2024-09-21T09:56:08.519 INFO:tasks.rados.rados.0.smithi033.stdout:1541: finishing write tid 5 to smithi03385152-15 2024-09-21T09:56:08.519 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 656 (ObjNum 506 snap 135 seq_num 506) dirty exists 2024-09-21T09:56:08.519 INFO:tasks.rados.rados.0.smithi033.stdout:1541: left oid 15 (ObjNum 506 snap 135 seq_num 506) 2024-09-21T09:56:08.519 INFO:tasks.rados.rados.0.smithi033.stdout:1544: seq_num 508 ranges {0=516096,516096=524288,1040384=278528} 2024-09-21T09:56:08.519 INFO:tasks.rados.rados.0.smithi033.stdout:1544: writing smithi03385152-48 from 0 to 516096 tid 1 2024-09-21T09:56:08.521 INFO:tasks.rados.rados.0.smithi033.stdout:1544: writing smithi03385152-48 from 516096 to 1040384 tid 2 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1544: writing smithi03385152-48 from 1040384 to 1318912 tid 3 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1541: done (3 left) 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1545: delete oid 46 current snap is 135 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1543: finishing rollback tid 1 to smithi03385152-28 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1542: finishing write tid 1 to smithi03385152-17 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1542: finishing write tid 2 to smithi03385152-17 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1542: finishing write tid 3 to smithi03385152-17 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1542: finishing write tid 4 to smithi03385152-17 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1544: finishing write tid 1 to smithi03385152-48 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:1542: finishing write tid 5 to smithi03385152-17 2024-09-21T09:56:08.524 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 553 (ObjNum 507 snap 135 seq_num 507) dirty exists 2024-09-21T09:56:08.525 INFO:tasks.rados.rados.0.smithi033.stdout:1542: left oid 17 (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:08.525 INFO:tasks.rados.rados.0.smithi033.stdout:1544: finishing write tid 2 to smithi03385152-48 2024-09-21T09:56:08.525 INFO:tasks.rados.rados.0.smithi033.stdout:1544: finishing write tid 3 to smithi03385152-48 2024-09-21T09:56:08.526 INFO:tasks.rados.rados.0.smithi033.stdout:1544: finishing write tid 4 to smithi03385152-48 2024-09-21T09:56:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1542: done (3 left) 2024-09-21T09:56:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1543: done (2 left) 2024-09-21T09:56:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1545: done (1 left) 2024-09-21T09:56:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 135 2024-09-21T09:56:08.527 INFO:tasks.rados.rados.0.smithi033.stdout:1546: seq_num 510 ranges {3235840=548864,3784704=745472,4530176=57344} 2024-09-21T09:56:08.548 INFO:tasks.rados.rados.0.smithi033.stdout:1546: writing smithi03385152-43 from 3235840 to 3784704 tid 1 2024-09-21T09:56:08.553 INFO:tasks.rados.rados.0.smithi033.stdout:1546: writing smithi03385152-43 from 3784704 to 4530176 tid 2 2024-09-21T09:56:08.553 INFO:tasks.rados.rados.0.smithi033.stdout:1546: writing smithi03385152-43 from 4530176 to 4587520 tid 3 2024-09-21T09:56:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:1547: read oid 2 snap -1 2024-09-21T09:56:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:1547: expect (ObjNum 400 snap 108 seq_num 400) 2024-09-21T09:56:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:1548: snap_create 2024-09-21T09:56:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:1544: finishing write tid 5 to smithi03385152-48 2024-09-21T09:56:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 853 (ObjNum 508 snap 135 seq_num 508) dirty exists 2024-09-21T09:56:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:1544: left oid 48 (ObjNum 508 snap 135 seq_num 508) 2024-09-21T09:56:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:1546: finishing write tid 1 to smithi03385152-43 2024-09-21T09:56:08.557 INFO:tasks.rados.rados.0.smithi033.stdout:1546: finishing write tid 2 to smithi03385152-43 2024-09-21T09:56:08.557 INFO:tasks.rados.rados.0.smithi033.stdout:1546: finishing write tid 3 to smithi03385152-43 2024-09-21T09:56:08.557 INFO:tasks.rados.rados.0.smithi033.stdout:1546: finishing write tid 4 to smithi03385152-43 2024-09-21T09:56:08.558 INFO:tasks.rados.rados.0.smithi033.stdout:1546: finishing write tid 5 to smithi03385152-43 2024-09-21T09:56:08.558 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 711 (ObjNum 510 snap 135 seq_num 510) dirty exists 2024-09-21T09:56:08.558 INFO:tasks.rados.rados.0.smithi033.stdout:1546: left oid 43 (ObjNum 510 snap 135 seq_num 510) 2024-09-21T09:56:08.563 INFO:tasks.rados.rados.0.smithi033.stdout:1547: expect (ObjNum 400 snap 108 seq_num 400) 2024-09-21T09:56:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:08 smithi033 ceph-mon[98546]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:56:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:56:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:08 smithi033 ceph-mon[98546]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:56:08.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:08 smithi033 ceph-mon[96568]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:56:08.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:56:08.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:08 smithi033 ceph-mon[96568]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:56:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:08 smithi105 ceph-mon[86649]: osdmap e341: 8 total, 8 up, 8 in 2024-09-21T09:56:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-21T09:56:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:08 smithi105 ceph-mon[86649]: osdmap e342: 8 total, 8 up, 8 in 2024-09-21T09:56:09.477 INFO:tasks.rados.rados.0.smithi033.stdout:1544: done (3 left) 2024-09-21T09:56:09.478 INFO:tasks.rados.rados.0.smithi033.stdout:1546: done (2 left) 2024-09-21T09:56:09.478 INFO:tasks.rados.rados.0.smithi033.stdout:1547: done (1 left) 2024-09-21T09:56:09.478 INFO:tasks.rados.rados.0.smithi033.stdout:1548: done (0 left) 2024-09-21T09:56:09.478 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 136 2024-09-21T09:56:09.478 INFO:tasks.rados.rados.0.smithi033.stdout:1549: seq_num 511 ranges {0=565248,565248=630784,1196032=172032} 2024-09-21T09:56:09.480 INFO:tasks.rados.rados.0.smithi033.stdout:1549: writing smithi03385152-27 from 0 to 565248 tid 1 2024-09-21T09:56:09.488 INFO:tasks.rados.rados.0.smithi033.stdout:1549: writing smithi03385152-27 from 565248 to 1196032 tid 2 2024-09-21T09:56:09.489 INFO:tasks.rados.rados.0.smithi033.stdout:1549: writing smithi03385152-27 from 1196032 to 1368064 tid 3 2024-09-21T09:56:09.489 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 136 2024-09-21T09:56:09.489 INFO:tasks.rados.rados.0.smithi033.stdout:1549: finishing write tid 1 to smithi03385152-27 2024-09-21T09:56:09.489 INFO:tasks.rados.rados.0.smithi033.stdout:1550: seq_num 512 ranges {3489792=581632,4071424=516096,4587520=286720} 2024-09-21T09:56:09.508 INFO:tasks.rados.rados.0.smithi033.stdout:1550: writing smithi03385152-15 from 3489792 to 4071424 tid 1 2024-09-21T09:56:09.511 INFO:tasks.rados.rados.0.smithi033.stdout:1550: writing smithi03385152-15 from 4071424 to 4587520 tid 2 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1550: writing smithi03385152-15 from 4587520 to 4874240 tid 3 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1551: read oid 29 snap -1 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1551: expect (ObjNum 504 snap 132 seq_num 504) 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 136 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1549: finishing write tid 2 to smithi03385152-27 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1549: finishing write tid 3 to smithi03385152-27 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1549: finishing write tid 4 to smithi03385152-27 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1549: finishing write tid 5 to smithi03385152-27 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 839 (ObjNum 511 snap 136 seq_num 511) dirty exists 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1549: left oid 27 (ObjNum 511 snap 136 seq_num 511) 2024-09-21T09:56:09.513 INFO:tasks.rados.rados.0.smithi033.stdout:1552: seq_num 513 ranges {0=598016,598016=794624,1392640=8192} 2024-09-21T09:56:09.516 INFO:tasks.rados.rados.0.smithi033.stdout:1552: writing smithi03385152-31 from 0 to 598016 tid 1 2024-09-21T09:56:09.523 INFO:tasks.rados.rados.0.smithi033.stdout:1552: writing smithi03385152-31 from 598016 to 1392640 tid 2 2024-09-21T09:56:09.523 INFO:tasks.rados.rados.0.smithi033.stdout:1552: writing smithi03385152-31 from 1392640 to 1400832 tid 3 2024-09-21T09:56:09.523 INFO:tasks.rados.rados.0.smithi033.stdout:1549: done (3 left) 2024-09-21T09:56:09.523 INFO:tasks.rados.rados.0.smithi033.stdout:1553: read oid 38 snap -1 2024-09-21T09:56:09.523 INFO:tasks.rados.rados.0.smithi033.stdout:1553: expect (ObjNum 499 snap 128 seq_num 499) 2024-09-21T09:56:09.523 INFO:tasks.rados.rados.0.smithi033.stdout:1554: snap_remove snap 133 2024-09-21T09:56:09.574 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:09 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:56:09.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:09 smithi033 ceph-mon[98546]: Upgrade: osd.1 is safe to restart 2024-09-21T09:56:09.575 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:09 smithi033 ceph-mon[98546]: pgmap v141: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-21T09:56:09.575 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:09 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:56:09.575 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:09 smithi033 ceph-mon[96568]: Upgrade: osd.1 is safe to restart 2024-09-21T09:56:09.575 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:09 smithi033 ceph-mon[96568]: pgmap v141: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-21T09:56:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:09 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-21T09:56:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:09 smithi105 ceph-mon[86649]: Upgrade: osd.1 is safe to restart 2024-09-21T09:56:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:09 smithi105 ceph-mon[86649]: pgmap v141: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.4 MiB/s wr, 26 op/s 2024-09-21T09:56:10.482 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:10.490 INFO:tasks.rados.rados.0.smithi033.stdout:1554: done (4 left) 2024-09-21T09:56:10.490 INFO:tasks.rados.rados.0.smithi033.stdout:1555: delete oid 36 current snap is 136 2024-09-21T09:56:10.490 INFO:tasks.rados.rados.0.smithi033.stdout:1550: finishing write tid 1 to smithi03385152-15 2024-09-21T09:56:10.490 INFO:tasks.rados.rados.0.smithi033.stdout:1550: finishing write tid 2 to smithi03385152-15 2024-09-21T09:56:10.490 INFO:tasks.rados.rados.0.smithi033.stdout:1550: finishing write tid 3 to smithi03385152-15 2024-09-21T09:56:10.490 INFO:tasks.rados.rados.0.smithi033.stdout:1550: finishing write tid 4 to smithi03385152-15 2024-09-21T09:56:10.490 INFO:tasks.rados.rados.0.smithi033.stdout:1550: finishing write tid 5 to smithi03385152-15 2024-09-21T09:56:10.491 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 660 (ObjNum 512 snap 136 seq_num 512) dirty exists 2024-09-21T09:56:10.491 INFO:tasks.rados.rados.0.smithi033.stdout:1550: left oid 15 (ObjNum 512 snap 136 seq_num 512) 2024-09-21T09:56:10.491 INFO:tasks.rados.rados.0.smithi033.stdout:1551: expect (ObjNum 504 snap 132 seq_num 504) 2024-09-21T09:56:10.560 INFO:tasks.rados.rados.0.smithi033.stdout:1552: finishing write tid 1 to smithi03385152-31 2024-09-21T09:56:10.560 INFO:tasks.rados.rados.0.smithi033.stdout:1552: finishing write tid 2 to smithi03385152-31 2024-09-21T09:56:10.560 INFO:tasks.rados.rados.0.smithi033.stdout:1552: finishing write tid 3 to smithi03385152-31 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:1550: done (4 left) 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:1551: done (3 left) 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:1555: done (2 left) 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:1556: setattr oid 29 current snap is 136 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:1552: finishing write tid 4 to smithi03385152-31 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:1552: finishing write tid 5 to smithi03385152-31 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 974 (ObjNum 513 snap 136 seq_num 513) dirty exists 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:1552: left oid 31 (ObjNum 513 snap 136 seq_num 513) 2024-09-21T09:56:10.561 INFO:tasks.rados.rados.0.smithi033.stdout:1553: expect (ObjNum 499 snap 128 seq_num 499) 2024-09-21T09:56:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[98546]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:56:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[98546]: Upgrade: Updating osd.1 2024-09-21T09:56:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:56:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:10.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[98546]: Deploying daemon osd.1 on smithi033 2024-09-21T09:56:10.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[96568]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:56:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[96568]: Upgrade: Updating osd.1 2024-09-21T09:56:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:56:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:10.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:10 smithi033 ceph-mon[96568]: Deploying daemon osd.1 on smithi033 2024-09-21T09:56:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:10 smithi105 ceph-mon[86649]: osdmap e343: 8 total, 8 up, 8 in 2024-09-21T09:56:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:10 smithi105 ceph-mon[86649]: Upgrade: Updating osd.1 2024-09-21T09:56:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:10 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:10 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-21T09:56:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:10 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:10.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:10 smithi105 ceph-mon[86649]: Deploying daemon osd.1 on smithi033 2024-09-21T09:56:11.217 INFO:tasks.rados.rados.0.smithi033.stdout:1552: done (2 left) 2024-09-21T09:56:11.217 INFO:tasks.rados.rados.0.smithi033.stdout:1553: done (1 left) 2024-09-21T09:56:11.217 INFO:tasks.rados.rados.0.smithi033.stdout:1557: read oid 3 snap 135 2024-09-21T09:56:11.218 INFO:tasks.rados.rados.0.smithi033.stdout:1557: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:56:11.218 INFO:tasks.rados.rados.0.smithi033.stdout:1558: copy_from oid 50 from oid 46 current snap is 136 2024-09-21T09:56:11.218 INFO:tasks.rados.rados.0.smithi033.stdout:1559: snap_create 2024-09-21T09:56:11.220 INFO:tasks.rados.rados.0.smithi033.stdout:1558: finishing copy_from to smithi03385152-50 2024-09-21T09:56:11.220 INFO:tasks.rados.rados.0.smithi033.stdout:1558: got expected ENOENT (src dne) 2024-09-21T09:56:11.220 INFO:tasks.rados.rados.0.smithi033.stdout:1558: finishing copy_from racing read to smithi03385152-50 2024-09-21T09:56:11.220 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 977 (ObjNum 504 snap 132 seq_num 504) dirty exists 2024-09-21T09:56:11.221 INFO:tasks.rados.rados.0.smithi033.stdout:1557: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:56:11.528 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:11 smithi033 ceph-mon[98546]: pgmap v144: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:11.528 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:11 smithi033 ceph-mon[98546]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:56:11.528 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:11 smithi033 ceph-mon[96568]: pgmap v144: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:11.528 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:11 smithi033 ceph-mon[96568]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:56:11.614 INFO:tasks.rados.rados.0.smithi033.stdout:1556: done (3 left) 2024-09-21T09:56:11.614 INFO:tasks.rados.rados.0.smithi033.stdout:1557: done (2 left) 2024-09-21T09:56:11.614 INFO:tasks.rados.rados.0.smithi033.stdout:1558: done (1 left) 2024-09-21T09:56:11.614 INFO:tasks.rados.rados.0.smithi033.stdout:1559: done (0 left) 2024-09-21T09:56:11.614 INFO:tasks.rados.rados.0.smithi033.stdout:1560: read oid 34 snap -1 2024-09-21T09:56:11.614 INFO:tasks.rados.rados.0.smithi033.stdout:1560: expect deleted 2024-09-21T09:56:11.615 INFO:tasks.rados.rados.0.smithi033.stdout:1561: setattr oid 5 current snap is 137 2024-09-21T09:56:11.615 INFO:tasks.rados.rados.0.smithi033.stdout:1562: snap_create 2024-09-21T09:56:11.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 206 (ObjNum 483 snap 127 seq_num 483) dirty exists 2024-09-21T09:56:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:11 smithi105 ceph-mon[86649]: pgmap v144: 121 pgs: 121 active+clean; 139 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:11 smithi105 ceph-mon[86649]: osdmap e344: 8 total, 8 up, 8 in 2024-09-21T09:56:11.865 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:56:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:56:11] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.51.0" 2024-09-21T09:56:12.626 INFO:tasks.rados.rados.0.smithi033.stdout:1560: done (2 left) 2024-09-21T09:56:12.626 INFO:tasks.rados.rados.0.smithi033.stdout:1561: done (1 left) 2024-09-21T09:56:12.626 INFO:tasks.rados.rados.0.smithi033.stdout:1562: done (0 left) 2024-09-21T09:56:12.626 INFO:tasks.rados.rados.0.smithi033.stdout:1563: delete oid 6 current snap is 138 2024-09-21T09:56:12.629 INFO:tasks.rados.rados.0.smithi033.stdout:1563: done (0 left) 2024-09-21T09:56:12.629 INFO:tasks.rados.rados.0.smithi033.stdout:1564: snap_create 2024-09-21T09:56:12.735 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:56:12.814 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:12 smithi033 ceph-mon[98546]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:56:12.815 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:12 smithi033 ceph-mon[96568]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:56:12.815 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:12 smithi033 systemd[1]: Stopping Ceph osd.1 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:56:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:12 smithi105 ceph-mon[86649]: osdmap e345: 8 total, 8 up, 8 in 2024-09-21T09:56:13.246 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:12 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[49350]: 2024-09-21T09:56:12.812+0000 7fd2d36fc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:56:13.246 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:12 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[49350]: 2024-09-21T09:56:12.812+0000 7fd2d36fc640 -1 osd.1 345 *** Got signal Terminated *** 2024-09-21T09:56:13.246 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:12 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[49350]: 2024-09-21T09:56:12.812+0000 7fd2d36fc640 -1 osd.1 345 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:56:13.331 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:56:13.331 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (3m) 24s ago 7m 20.1M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:56:13.331 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (2m) 65s ago 7m 82.4M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:56:13.331 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (9m) 24s ago 9m 15.8M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:56:13.331 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (9m) 65s ago 9m 19.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:56:13.331 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (2m) 65s ago 12m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:56:13.331 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (2m) 24s ago 14m 517M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (110s) 24s ago 14m 47.6M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (70s) 65s ago 12m 22.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (88s) 24s ago 12m 38.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (3m) 24s ago 6m 12.6M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (2m) 65s ago 6m 9915k - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (28s) 24s ago 12m 14.5M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (11m) 24s ago 11m 204M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 756b9c068be4 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (11m) 24s ago 11m 173M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (10m) 24s ago 10m 105M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (10m) 65s ago 10m 167M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (10m) 65s ago 10m 193M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (9m) 65s ago 9m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (9m) 65s ago 9m 140M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:56:13.332 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (2m) 65s ago 7m 44.0M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:56:13.451 INFO:tasks.rados.rados.0.smithi033.stdout:1564: done (0 left) 2024-09-21T09:56:13.452 INFO:tasks.rados.rados.0.smithi033.stdout:1565: snap_remove snap 129 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 6, 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 1 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:56:13.695 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:56:13.696 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:56:13.696 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:56:13.696 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8, 2024-09-21T09:56:13.696 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 6 2024-09-21T09:56:13.696 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:56:13.696 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[98546]: pgmap v147: 121 pgs: 121 active+clean; 148 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[98546]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[98546]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[98546]: osd.1 marked itself down and dead 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[98546]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[98546]: osdmap e347: 8 total, 7 up, 8 in 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[96568]: pgmap v147: 121 pgs: 121 active+clean; 148 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[96568]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:56:13.993 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[96568]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:13.994 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[96568]: osd.1 marked itself down and dead 2024-09-21T09:56:13.994 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[96568]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:56:13.994 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:13 smithi033 ceph-mon[96568]: osdmap e347: 8 total, 7 up, 8 in 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "6/20 daemons upgraded", 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:56:13.999 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:56:14.000 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:56:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:13 smithi105 ceph-mon[86649]: pgmap v147: 121 pgs: 121 active+clean; 148 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-09-21T09:56:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:13 smithi105 ceph-mon[86649]: osdmap e346: 8 total, 8 up, 8 in 2024-09-21T09:56:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:13 smithi105 ceph-mon[86649]: from='client.34254 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:13 smithi105 ceph-mon[86649]: osd.1 marked itself down and dead 2024-09-21T09:56:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:13 smithi105 ceph-mon[86649]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:56:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:13 smithi105 ceph-mon[86649]: osdmap e347: 8 total, 7 up, 8 in 2024-09-21T09:56:14.246 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:13 smithi033 podman[107852]: 2024-09-21 09:56:13.990926499 +0000 UTC m=+1.309967216 container died 756b9c068be4e4a5ae5778e27fecdc67254c0ca1c81d24586b2d437d4ac60f78 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:14.456 INFO:tasks.rados.rados.0.smithi033.stdout:1565: done (0 left) 2024-09-21T09:56:14.456 INFO:tasks.rados.rados.0.smithi033.stdout:1566: delete oid 32 current snap is 139 2024-09-21T09:56:14.471 INFO:tasks.rados.rados.0.smithi033.stdout:1566: done (0 left) 2024-09-21T09:56:14.471 INFO:tasks.rados.rados.0.smithi033.stdout:1567: delete oid 21 current snap is 139 2024-09-21T09:56:14.474 INFO:tasks.rados.rados.0.smithi033.stdout:1567: done (0 left) 2024-09-21T09:56:14.475 INFO:tasks.rados.rados.0.smithi033.stdout:1568: snap_remove snap 135 2024-09-21T09:56:14.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:14 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:14.623 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:14 smithi033 podman[107852]: 2024-09-21 09:56:14.343376043 +0000 UTC m=+1.662416752 container cleanup 756b9c068be4e4a5ae5778e27fecdc67254c0ca1c81d24586b2d437d4ac60f78 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T09:56:14.623 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:14 smithi033 bash[107852]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1 2024-09-21T09:56:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:14 smithi033 ceph-mon[98546]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:14 smithi033 ceph-mon[98546]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:14 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2806560762' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:14 smithi033 ceph-mon[98546]: osdmap e348: 8 total, 7 up, 8 in 2024-09-21T09:56:14.996 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:14 smithi033 podman[107963]: 2024-09-21 09:56:14.655789732 +0000 UTC m=+0.659585531 container remove 756b9c068be4e4a5ae5778e27fecdc67254c0ca1c81d24586b2d437d4ac60f78 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=quincy-fbd1a6a, ceph=True, CEPH_POINT_RELEASE=) 2024-09-21T09:56:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:14 smithi033 ceph-mon[96568]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:14.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:14 smithi033 ceph-mon[96568]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:14.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:14 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2806560762' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:14.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:14 smithi033 ceph-mon[96568]: osdmap e348: 8 total, 7 up, 8 in 2024-09-21T09:56:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:14 smithi105 ceph-mon[86649]: from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:14 smithi105 ceph-mon[86649]: from='client.44180 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:14 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2806560762' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:14 smithi105 ceph-mon[86649]: osdmap e348: 8 total, 7 up, 8 in 2024-09-21T09:56:15.350 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:15 smithi033 podman[108066]: 2024-09-21 09:56:14.952667243 +0000 UTC m=+0.017306509 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:56:15.350 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:15 smithi033 podman[108066]: 2024-09-21 09:56:15.054383456 +0000 UTC m=+0.119022715 container create 55a4de99d608e26471762943f52e2b1d8037d997f70d2d9c83b16de58a1ed8c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T09:56:15.455 INFO:tasks.rados.rados.0.smithi033.stdout:1568: done (0 left) 2024-09-21T09:56:15.456 INFO:tasks.rados.rados.0.smithi033.stdout:1569: snap_remove snap 138 2024-09-21T09:56:15.625 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:15 smithi033 podman[108066]: 2024-09-21 09:56:15.348675065 +0000 UTC m=+0.413314327 container init 55a4de99d608e26471762943f52e2b1d8037d997f70d2d9c83b16de58a1ed8c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:56:15.625 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:15 smithi033 podman[108066]: 2024-09-21 09:56:15.354005275 +0000 UTC m=+0.418644532 container start 55a4de99d608e26471762943f52e2b1d8037d997f70d2d9c83b16de58a1ed8c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:56:15.625 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:15 smithi033 podman[108066]: 2024-09-21 09:56:15.448762479 +0000 UTC m=+0.513401743 container attach 55a4de99d608e26471762943f52e2b1d8037d997f70d2d9c83b16de58a1ed8c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:56:15.625 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:15 smithi033 podman[108066]: 2024-09-21 09:56:15.553643494 +0000 UTC m=+0.618282769 container died 55a4de99d608e26471762943f52e2b1d8037d997f70d2d9c83b16de58a1ed8c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:56:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:15 smithi033 ceph-mon[98546]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:15 smithi033 ceph-mon[98546]: pgmap v150: 121 pgs: 16 stale+active+clean, 105 active+clean; 148 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-09-21T09:56:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:15 smithi033 ceph-mon[98546]: osdmap e349: 8 total, 7 up, 8 in 2024-09-21T09:56:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:15 smithi033 ceph-mon[96568]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:15.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:15 smithi033 ceph-mon[96568]: pgmap v150: 121 pgs: 16 stale+active+clean, 105 active+clean; 148 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-09-21T09:56:15.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:15 smithi033 ceph-mon[96568]: osdmap e349: 8 total, 7 up, 8 in 2024-09-21T09:56:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:15 smithi105 ceph-mon[86649]: from='client.44192 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:15 smithi105 ceph-mon[86649]: pgmap v150: 121 pgs: 16 stale+active+clean, 105 active+clean; 148 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.2 MiB/s wr, 30 op/s 2024-09-21T09:56:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:15 smithi105 ceph-mon[86649]: osdmap e349: 8 total, 7 up, 8 in 2024-09-21T09:56:16.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:16.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:16.451 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 podman[108113]: 2024-09-21 09:56:16.15527771 +0000 UTC m=+0.594913860 container remove 55a4de99d608e26471762943f52e2b1d8037d997f70d2d9c83b16de58a1ed8c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T09:56:16.451 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.1.service: Deactivated successfully. 2024-09-21T09:56:16.451 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 systemd[1]: Stopped Ceph osd.1 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:56:16.451 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.1.service: Consumed 12.707s CPU time. 2024-09-21T09:56:16.451 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 systemd[1]: Starting Ceph osd.1 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:56:16.456 INFO:tasks.rados.rados.0.smithi033.stdout:1569: done (0 left) 2024-09-21T09:56:16.456 INFO:tasks.rados.rados.0.smithi033.stdout:1570: copy_from oid 11 from oid 16 current snap is 139 2024-09-21T09:56:16.456 INFO:tasks.rados.rados.0.smithi033.stdout:1571: rollback oid 40 current snap is 139 2024-09-21T09:56:16.456 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 40 to 134 2024-09-21T09:56:16.456 INFO:tasks.rados.rados.0.smithi033.stdout:1572: read oid 17 snap -1 2024-09-21T09:56:16.456 INFO:tasks.rados.rados.0.smithi033.stdout:1572: expect (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:16.456 INFO:tasks.rados.rados.0.smithi033.stdout:1573: copy_from oid 10 from oid 2 current snap is 139 2024-09-21T09:56:16.457 INFO:tasks.rados.rados.0.smithi033.stdout:1574: snap_remove snap 136 2024-09-21T09:56:16.746 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 podman[108217]: 2024-09-21 09:56:16.662139186 +0000 UTC m=+0.103081267 container create 589d7618a99e3e4ef884519da0b57730ca92f377f84108b89d400ec296d3e922 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:16.746 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 podman[108217]: 2024-09-21 09:56:16.575884391 +0000 UTC m=+0.016826470 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:56:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:16 smithi105 ceph-mon[86649]: osdmap e350: 8 total, 7 up, 8 in 2024-09-21T09:56:17.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:16 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:17.039 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:17.039 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:17.039 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:16 smithi033 ceph-mon[98546]: osdmap e350: 8 total, 7 up, 8 in 2024-09-21T09:56:17.039 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 podman[108217]: 2024-09-21 09:56:16.897992979 +0000 UTC m=+0.338935065 container init 589d7618a99e3e4ef884519da0b57730ca92f377f84108b89d400ec296d3e922 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T09:56:17.039 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 podman[108217]: 2024-09-21 09:56:16.902874607 +0000 UTC m=+0.343816693 container start 589d7618a99e3e4ef884519da0b57730ca92f377f84108b89d400ec296d3e922 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=main-62da54b, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T09:56:17.039 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:16 smithi033 podman[108217]: 2024-09-21 09:56:16.981175601 +0000 UTC m=+0.422117680 container attach 589d7618a99e3e4ef884519da0b57730ca92f377f84108b89d400ec296d3e922 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2) 2024-09-21T09:56:17.039 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:17.040 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:17.040 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:16 smithi033 ceph-mon[96568]: osdmap e350: 8 total, 7 up, 8 in 2024-09-21T09:56:17.457 INFO:tasks.rados.rados.0.smithi033.stdout:1574: done (4 left) 2024-09-21T09:56:17.457 INFO:tasks.rados.rados.0.smithi033.stdout:1575: snap_create 2024-09-21T09:56:17.457 INFO:tasks.rados.rados.0.smithi033.stdout:1571: finishing rollback tid 0 to smithi03385152-40 2024-09-21T09:56:17.457 INFO:tasks.rados.rados.0.smithi033.stdout:1571: finishing rollback tid 1 to smithi03385152-40 2024-09-21T09:56:17.457 INFO:tasks.rados.rados.0.smithi033.stdout:1571: finishing rollback tid 2 to smithi03385152-40 2024-09-21T09:56:17.457 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 723 (ObjNum 459 snap 121 seq_num 459) dirty exists 2024-09-21T09:56:17.457 INFO:tasks.rados.rados.0.smithi033.stdout:1570: finishing copy_from to smithi03385152-11 2024-09-21T09:56:17.458 INFO:tasks.rados.rados.0.smithi033.stdout:1570: got expected ENOENT (src dne) 2024-09-21T09:56:17.458 INFO:tasks.rados.rados.0.smithi033.stdout:1570: finishing copy_from racing read to smithi03385152-11 2024-09-21T09:56:17.458 INFO:tasks.rados.rados.0.smithi033.stdout:1573: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:56:17.458 INFO:tasks.rados.rados.0.smithi033.stdout:1573: finishing copy_from to smithi03385152-10 2024-09-21T09:56:17.458 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 624 (ObjNum 400 snap 108 seq_num 400) dirty exists 2024-09-21T09:56:17.458 INFO:tasks.rados.rados.0.smithi033.stdout:1572: expect (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:17.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:56:17.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:56:17.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:56:17.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:56:17.809 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-mon[96568]: pgmap v153: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 6 op/s; 38/258 objects degraded (14.729%) 2024-09-21T09:56:17.809 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-mon[96568]: Health check failed: Degraded data redundancy: 38/258 objects degraded (14.729%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:17.809 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-mon[96568]: osdmap e351: 8 total, 7 up, 8 in 2024-09-21T09:56:17.809 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-mon[98546]: pgmap v153: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 6 op/s; 38/258 objects degraded (14.729%) 2024-09-21T09:56:17.809 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-mon[98546]: Health check failed: Degraded data redundancy: 38/258 objects degraded (14.729%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:17.809 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-mon[98546]: osdmap e351: 8 total, 7 up, 8 in 2024-09-21T09:56:17.810 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:56:17.810 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:56:17.810 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:56:17.810 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:56:17.810 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:56:17.810 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:56:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:17 smithi105 ceph-mon[86649]: pgmap v153: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 6 op/s; 38/258 objects degraded (14.729%) 2024-09-21T09:56:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:17 smithi105 ceph-mon[86649]: Health check failed: Degraded data redundancy: 38/258 objects degraded (14.729%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:17 smithi105 ceph-mon[86649]: osdmap e351: 8 total, 7 up, 8 in 2024-09-21T09:56:18.202 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:56:18.203 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:56:18.203 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T09:56:18.203 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-21T09:56:18.203 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:56:18.203 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-21T09:56:18.203 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate[108233]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T09:56:18.203 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 bash[108217]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-21T09:56:18.203 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:17 smithi033 podman[108217]: 2024-09-21 09:56:17.837327891 +0000 UTC m=+1.278269986 container died 589d7618a99e3e4ef884519da0b57730ca92f377f84108b89d400ec296d3e922 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-21T09:56:18.460 INFO:tasks.rados.rados.0.smithi033.stdout:1570: done (4 left) 2024-09-21T09:56:18.460 INFO:tasks.rados.rados.0.smithi033.stdout:1571: done (3 left) 2024-09-21T09:56:18.461 INFO:tasks.rados.rados.0.smithi033.stdout:1572: done (2 left) 2024-09-21T09:56:18.461 INFO:tasks.rados.rados.0.smithi033.stdout:1573: done (1 left) 2024-09-21T09:56:18.461 INFO:tasks.rados.rados.0.smithi033.stdout:1575: done (0 left) 2024-09-21T09:56:18.461 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 140 2024-09-21T09:56:18.461 INFO:tasks.rados.rados.0.smithi033.stdout:1576: seq_num 518 ranges {1794048=679936,2473984=671744,3145728=131072} 2024-09-21T09:56:18.473 INFO:tasks.rados.rados.0.smithi033.stdout:1576: writing smithi03385152-49 from 1794048 to 2473984 tid 1 2024-09-21T09:56:18.477 INFO:tasks.rados.rados.0.smithi033.stdout:1576: writing smithi03385152-49 from 2473984 to 3145728 tid 2 2024-09-21T09:56:18.478 INFO:tasks.rados.rados.0.smithi033.stdout:1576: writing smithi03385152-49 from 3145728 to 3276800 tid 3 2024-09-21T09:56:18.478 INFO:tasks.rados.rados.0.smithi033.stdout:1577: setattr oid 31 current snap is 140 2024-09-21T09:56:18.478 INFO:tasks.rados.rados.0.smithi033.stdout:1576: finishing write tid 1 to smithi03385152-49 2024-09-21T09:56:18.480 INFO:tasks.rados.rados.0.smithi033.stdout:1578: read oid 35 snap -1 2024-09-21T09:56:18.480 INFO:tasks.rados.rados.0.smithi033.stdout:1578: expect (ObjNum 500 snap 131 seq_num 500) 2024-09-21T09:56:18.480 INFO:tasks.rados.rados.0.smithi033.stdout:1579: snap_remove snap 137 2024-09-21T09:56:18.961 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:18 smithi033 podman[108383]: 2024-09-21 09:56:18.613916673 +0000 UTC m=+0.770247597 container remove 589d7618a99e3e4ef884519da0b57730ca92f377f84108b89d400ec296d3e922 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T09:56:18.961 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:18 smithi033 podman[108449]: 2024-09-21 09:56:18.727572859 +0000 UTC m=+0.016189080 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:56:18.961 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:18 smithi033 podman[108449]: 2024-09-21 09:56:18.853159358 +0000 UTC m=+0.141775604 container create 1deebc8df60c3dda39de98090ae694322c515f61dead6b94491a8a143b1d3040 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:1579: done (3 left) 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:1580: setattr oid 18 current snap is 140 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:1576: finishing write tid 2 to smithi03385152-49 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:1576: finishing write tid 3 to smithi03385152-49 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:1576: finishing write tid 4 to smithi03385152-49 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:1576: finishing write tid 5 to smithi03385152-49 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 560 (ObjNum 518 snap 140 seq_num 518) dirty exists 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:1576: left oid 49 (ObjNum 518 snap 140 seq_num 518) 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 979 (ObjNum 513 snap 136 seq_num 513) dirty exists 2024-09-21T09:56:19.463 INFO:tasks.rados.rados.0.smithi033.stdout:1578: expect (ObjNum 500 snap 131 seq_num 500) 2024-09-21T09:56:19.466 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:19 smithi033 podman[108449]: 2024-09-21 09:56:19.038797674 +0000 UTC m=+0.327413878 container init 1deebc8df60c3dda39de98090ae694322c515f61dead6b94491a8a143b1d3040 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T09:56:19.466 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:19 smithi033 podman[108449]: 2024-09-21 09:56:19.043889638 +0000 UTC m=+0.332505843 container start 1deebc8df60c3dda39de98090ae694322c515f61dead6b94491a8a143b1d3040 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T09:56:19.466 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-osd[108487]: -- 172.21.15.33:0/3784822984 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561f23fdba40 con 0x561f24daa400 2024-09-21T09:56:19.466 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:19 smithi033 bash[108449]: 1deebc8df60c3dda39de98090ae694322c515f61dead6b94491a8a143b1d3040 2024-09-21T09:56:19.466 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:19 smithi033 systemd[1]: Started Ceph osd.1 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:56:19.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[96568]: pgmap v156: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 6 op/s; 38/258 objects degraded (14.729%) 2024-09-21T09:56:19.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[96568]: osdmap e352: 8 total, 7 up, 8 in 2024-09-21T09:56:19.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:19.466 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:19.467 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:1576: done (3 left) 2024-09-21T09:56:19.497 INFO:tasks.rados.rados.0.smithi033.stdout:1577: done (2 left) 2024-09-21T09:56:19.497 INFO:tasks.rados.rados.0.smithi033.stdout:1578: done (1 left) 2024-09-21T09:56:19.497 INFO:tasks.rados.rados.0.smithi033.stdout:1581: snap_create 2024-09-21T09:56:19.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 129 (ObjNum 177 snap 0 seq_num 1163894144) dirty exists 2024-09-21T09:56:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[98546]: pgmap v156: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 6 op/s; 38/258 objects degraded (14.729%) 2024-09-21T09:56:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[98546]: osdmap e352: 8 total, 7 up, 8 in 2024-09-21T09:56:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:19 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:19 smithi105 ceph-mon[86649]: pgmap v156: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 6 op/s; 38/258 objects degraded (14.729%) 2024-09-21T09:56:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:19 smithi105 ceph-mon[86649]: osdmap e352: 8 total, 7 up, 8 in 2024-09-21T09:56:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:19 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:19 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:19 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:20.470 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[108464]: 2024-09-21T09:56:20.194+0000 7f420e76d640 -1 Falling back to public interface 2024-09-21T09:56:20.471 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:20.474 INFO:tasks.rados.rados.0.smithi033.stdout:1580: done (1 left) 2024-09-21T09:56:20.474 INFO:tasks.rados.rados.0.smithi033.stdout:1581: done (0 left) 2024-09-21T09:56:20.474 INFO:tasks.rados.rados.0.smithi033.stdout:1582: snap_create 2024-09-21T09:56:20.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:20 smithi033 ceph-mon[98546]: osdmap e353: 8 total, 7 up, 8 in 2024-09-21T09:56:20.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:20 smithi033 ceph-mon[96568]: osdmap e353: 8 total, 7 up, 8 in 2024-09-21T09:56:20.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:20 smithi105 ceph-mon[86649]: osdmap e353: 8 total, 7 up, 8 in 2024-09-21T09:56:21.476 INFO:tasks.rados.rados.0.smithi033.stdout:1582: done (0 left) 2024-09-21T09:56:21.476 INFO:tasks.rados.rados.0.smithi033.stdout:1583: snap_remove snap 132 2024-09-21T09:56:21.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:21 smithi033 ceph-mon[96568]: pgmap v159: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38/258 objects degraded (14.729%) 2024-09-21T09:56:21.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:21 smithi033 ceph-mon[96568]: osdmap e354: 8 total, 7 up, 8 in 2024-09-21T09:56:21.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:21 smithi033 ceph-mon[98546]: pgmap v159: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38/258 objects degraded (14.729%) 2024-09-21T09:56:21.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:21 smithi033 ceph-mon[98546]: osdmap e354: 8 total, 7 up, 8 in 2024-09-21T09:56:21.747 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:56:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:56:21] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.51.0" 2024-09-21T09:56:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:21 smithi105 ceph-mon[86649]: pgmap v159: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 145 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38/258 objects degraded (14.729%) 2024-09-21T09:56:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:21 smithi105 ceph-mon[86649]: osdmap e354: 8 total, 7 up, 8 in 2024-09-21T09:56:22.476 INFO:tasks.rados.rados.0.smithi033.stdout:1583: done (0 left) 2024-09-21T09:56:22.476 INFO:tasks.rados.rados.0.smithi033.stdout:1584: rollback oid 7 current snap is 142 2024-09-21T09:56:22.476 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 7 to 131 2024-09-21T09:56:22.477 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 142 2024-09-21T09:56:22.477 INFO:tasks.rados.rados.0.smithi033.stdout:1585: seq_num 519 ranges {1400832=696320,2097152=548864,2646016=253952} 2024-09-21T09:56:22.491 INFO:tasks.rados.rados.0.smithi033.stdout:1585: writing smithi03385152-31 from 1400832 to 2097152 tid 1 2024-09-21T09:56:22.494 INFO:tasks.rados.rados.0.smithi033.stdout:1585: writing smithi03385152-31 from 2097152 to 2646016 tid 2 2024-09-21T09:56:22.496 INFO:tasks.rados.rados.0.smithi033.stdout:1585: writing smithi03385152-31 from 2646016 to 2899968 tid 3 2024-09-21T09:56:22.496 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 142 2024-09-21T09:56:22.496 INFO:tasks.rados.rados.0.smithi033.stdout:1584: finishing rollback tid 0 to smithi03385152-7 2024-09-21T09:56:22.496 INFO:tasks.rados.rados.0.smithi033.stdout:1586: seq_num 520 ranges {0=712704,712704=434176,1146880=368640} 2024-09-21T09:56:22.500 INFO:tasks.rados.rados.0.smithi033.stdout:1586: writing smithi03385152-50 from 0 to 712704 tid 1 2024-09-21T09:56:22.503 INFO:tasks.rados.rados.0.smithi033.stdout:1586: writing smithi03385152-50 from 712704 to 1146880 tid 2 2024-09-21T09:56:22.505 INFO:tasks.rados.rados.0.smithi033.stdout:1586: writing smithi03385152-50 from 1146880 to 1515520 tid 3 2024-09-21T09:56:22.505 INFO:tasks.rados.rados.0.smithi033.stdout:1587: read oid 9 snap -1 2024-09-21T09:56:22.505 INFO:tasks.rados.rados.0.smithi033.stdout:1587: expect (ObjNum 479 snap 125 seq_num 479) 2024-09-21T09:56:22.505 INFO:tasks.rados.rados.0.smithi033.stdout:1588: snap_remove snap 140 2024-09-21T09:56:22.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:22 smithi033 ceph-mon[98546]: osdmap e355: 8 total, 7 up, 8 in 2024-09-21T09:56:22.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:22 smithi033 ceph-mon[98546]: osdmap e356: 8 total, 7 up, 8 in 2024-09-21T09:56:22.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:22 smithi033 ceph-mon[96568]: osdmap e355: 8 total, 7 up, 8 in 2024-09-21T09:56:22.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:22 smithi033 ceph-mon[96568]: osdmap e356: 8 total, 7 up, 8 in 2024-09-21T09:56:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:22 smithi105 ceph-mon[86649]: osdmap e355: 8 total, 7 up, 8 in 2024-09-21T09:56:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:22 smithi105 ceph-mon[86649]: osdmap e356: 8 total, 7 up, 8 in 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1588: done (4 left) 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1589: copy_from oid 49 from oid 27 current snap is 142 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1584: finishing rollback tid 1 to smithi03385152-7 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1584: finishing rollback tid 2 to smithi03385152-7 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 983 (ObjNum 474 snap 125 seq_num 474) dirty exists 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1585: finishing write tid 1 to smithi03385152-31 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1585: finishing write tid 2 to smithi03385152-31 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1585: finishing write tid 3 to smithi03385152-31 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1585: finishing write tid 4 to smithi03385152-31 2024-09-21T09:56:23.497 INFO:tasks.rados.rados.0.smithi033.stdout:1585: finishing write tid 5 to smithi03385152-31 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 988 (ObjNum 519 snap 142 seq_num 519) dirty exists 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1585: left oid 31 (ObjNum 519 snap 142 seq_num 519) 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1586: finishing write tid 1 to smithi03385152-50 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1586: finishing write tid 2 to smithi03385152-50 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1586: finishing write tid 3 to smithi03385152-50 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1586: finishing write tid 4 to smithi03385152-50 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1584: done (4 left) 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1585: done (3 left) 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1590: setattr oid 5 current snap is 142 2024-09-21T09:56:23.498 INFO:tasks.rados.rados.0.smithi033.stdout:1587: expect (ObjNum 479 snap 125 seq_num 479) 2024-09-21T09:56:23.559 INFO:tasks.rados.rados.0.smithi033.stdout:1586: finishing write tid 5 to smithi03385152-50 2024-09-21T09:56:23.559 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 645 (ObjNum 520 snap 142 seq_num 520) dirty exists 2024-09-21T09:56:23.559 INFO:tasks.rados.rados.0.smithi033.stdout:1586: left oid 50 (ObjNum 520 snap 142 seq_num 520) 2024-09-21T09:56:23.559 INFO:tasks.rados.rados.0.smithi033.stdout:1589: finishing copy_from racing read to smithi03385152-49 2024-09-21T09:56:23.559 INFO:tasks.rados.rados.0.smithi033.stdout:1589: finishing copy_from to smithi03385152-49 2024-09-21T09:56:23.560 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 839 (ObjNum 511 snap 136 seq_num 511) dirty exists 2024-09-21T09:56:23.560 INFO:tasks.rados.rados.0.smithi033.stdout:1586: done (3 left) 2024-09-21T09:56:23.560 INFO:tasks.rados.rados.0.smithi033.stdout:1587: done (2 left) 2024-09-21T09:56:23.560 INFO:tasks.rados.rados.0.smithi033.stdout:1589: done (1 left) 2024-09-21T09:56:23.560 INFO:tasks.rados.rados.0.smithi033.stdout:1591: copy_from oid 15 from oid 32 current snap is 142 2024-09-21T09:56:23.560 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 142 2024-09-21T09:56:23.560 INFO:tasks.rados.rados.0.smithi033.stdout:1592: seq_num 521 ranges {13860864=729088,14589952=720896,15310848=81920} 2024-09-21T09:56:23.638 INFO:tasks.rados.rados.0.smithi033.stdout:1592: writing smithi03385152-24 from 13860864 to 14589952 tid 1 2024-09-21T09:56:23.643 INFO:tasks.rados.rados.0.smithi033.stdout:1592: writing smithi03385152-24 from 14589952 to 15310848 tid 2 2024-09-21T09:56:23.644 INFO:tasks.rados.rados.0.smithi033.stdout:1592: writing smithi03385152-24 from 15310848 to 15392768 tid 3 2024-09-21T09:56:23.644 INFO:tasks.rados.rados.0.smithi033.stdout:1591: finishing copy_from to smithi03385152-15 2024-09-21T09:56:23.644 INFO:tasks.rados.rados.0.smithi033.stdout:1591: got expected ENOENT (src dne) 2024-09-21T09:56:23.644 INFO:tasks.rados.rados.0.smithi033.stdout:1591: finishing copy_from racing read to smithi03385152-15 2024-09-21T09:56:23.644 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 216 (ObjNum 483 snap 127 seq_num 483) dirty exists 2024-09-21T09:56:23.644 INFO:tasks.rados.rados.0.smithi033.stdout:1590: done (2 left) 2024-09-21T09:56:23.644 INFO:tasks.rados.rados.0.smithi033.stdout:1591: done (1 left) 2024-09-21T09:56:23.644 INFO:tasks.rados.rados.0.smithi033.stdout:1593: snap_remove snap 141 2024-09-21T09:56:23.746 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[108464]: 2024-09-21T09:56:23.701+0000 7f420e76d640 -1 osd.1 346 log_to_monitors true 2024-09-21T09:56:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[98546]: pgmap v162: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 150 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 42/273 objects degraded (15.385%) 2024-09-21T09:56:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 42/273 objects degraded (15.385%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[96568]: pgmap v162: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 150 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 42/273 objects degraded (15.385%) 2024-09-21T09:56:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 42/273 objects degraded (15.385%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:23 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:23 smithi105 ceph-mon[86649]: pgmap v162: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 150 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 42/273 objects degraded (15.385%) 2024-09-21T09:56:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:23 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 42/273 objects degraded (15.385%), 12 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:23 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:23 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:23 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:23 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:24.500 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:24.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:56:24.500 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:24.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.1\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.1\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:24.502 INFO:tasks.rados.rados.0.smithi033.stdout:1593: done (1 left) 2024-09-21T09:56:24.502 INFO:tasks.rados.rados.0.smithi033.stdout:1594: copy_from oid 41 from oid 34 current snap is 142 2024-09-21T09:56:24.502 INFO:tasks.rados.rados.0.smithi033.stdout:1592: finishing write tid 1 to smithi03385152-24 2024-09-21T09:56:24.502 INFO:tasks.rados.rados.0.smithi033.stdout:1592: finishing write tid 2 to smithi03385152-24 2024-09-21T09:56:24.502 INFO:tasks.rados.rados.0.smithi033.stdout:1592: finishing write tid 3 to smithi03385152-24 2024-09-21T09:56:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:1592: finishing write tid 4 to smithi03385152-24 2024-09-21T09:56:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:1592: finishing write tid 5 to smithi03385152-24 2024-09-21T09:56:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 662 (ObjNum 521 snap 142 seq_num 521) dirty exists 2024-09-21T09:56:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:1592: left oid 24 (ObjNum 521 snap 142 seq_num 521) 2024-09-21T09:56:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:1592: done (1 left) 2024-09-21T09:56:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:1595: delete oid 50 current snap is 142 2024-09-21T09:56:24.508 INFO:tasks.rados.rados.0.smithi033.stdout:1594: finishing copy_from to smithi03385152-41 2024-09-21T09:56:24.509 INFO:tasks.rados.rados.0.smithi033.stdout:1594: got expected ENOENT (src dne) 2024-09-21T09:56:24.509 INFO:tasks.rados.rados.0.smithi033.stdout:1594: finishing copy_from racing read to smithi03385152-41 2024-09-21T09:56:24.512 INFO:tasks.rados.rados.0.smithi033.stdout:1594: done (1 left) 2024-09-21T09:56:24.512 INFO:tasks.rados.rados.0.smithi033.stdout:1595: done (0 left) 2024-09-21T09:56:24.512 INFO:tasks.rados.rados.0.smithi033.stdout:1596: rollback oid 41 current snap is 142 2024-09-21T09:56:24.512 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 131 2024-09-21T09:56:24.512 INFO:tasks.rados.rados.0.smithi033.stdout:1597: read oid 42 snap -1 2024-09-21T09:56:24.512 INFO:tasks.rados.rados.0.smithi033.stdout:1597: expect (ObjNum 427 snap 113 seq_num 427) 2024-09-21T09:56:24.512 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 142 2024-09-21T09:56:24.512 INFO:tasks.rados.rados.0.smithi033.stdout:1598: seq_num 523 ranges {1179648=770048,1949696=491520,2441216=311296} 2024-09-21T09:56:24.523 INFO:tasks.rados.rados.0.smithi033.stdout:1598: writing smithi03385152-35 from 1179648 to 1949696 tid 1 2024-09-21T09:56:24.526 INFO:tasks.rados.rados.0.smithi033.stdout:1598: writing smithi03385152-35 from 1949696 to 2441216 tid 2 2024-09-21T09:56:24.529 INFO:tasks.rados.rados.0.smithi033.stdout:1598: writing smithi03385152-35 from 2441216 to 2752512 tid 3 2024-09-21T09:56:24.529 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 142 2024-09-21T09:56:24.529 INFO:tasks.rados.rados.0.smithi033.stdout:1599: seq_num 524 ranges {1957888=786432,2744320=778240,3522560=24576} 2024-09-21T09:56:24.544 INFO:tasks.rados.rados.0.smithi033.stdout:1599: writing smithi03385152-29 from 1957888 to 2744320 tid 1 2024-09-21T09:56:24.549 INFO:tasks.rados.rados.0.smithi033.stdout:1599: writing smithi03385152-29 from 2744320 to 3522560 tid 2 2024-09-21T09:56:24.549 INFO:tasks.rados.rados.0.smithi033.stdout:1599: writing smithi03385152-29 from 3522560 to 3547136 tid 3 2024-09-21T09:56:24.549 INFO:tasks.rados.rados.0.smithi033.stdout:1600: read oid 4 snap -1 2024-09-21T09:56:24.549 INFO:tasks.rados.rados.0.smithi033.stdout:1600: expect (ObjNum 502 snap 132 seq_num 502) 2024-09-21T09:56:24.549 INFO:tasks.rados.rados.0.smithi033.stdout:1601: snap_create 2024-09-21T09:56:24.549 INFO:tasks.rados.rados.0.smithi033.stdout:1596: finishing rollback tid 1 to smithi03385152-41 2024-09-21T09:56:24.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 720 (ObjNum 33 snap 0 seq_num 90918272) dirty dne 2024-09-21T09:56:24.549 INFO:tasks.rados.rados.0.smithi033.stdout:1597: expect (ObjNum 427 snap 113 seq_num 427) 2024-09-21T09:56:24.618 INFO:tasks.rados.rados.0.smithi033.stdout:1598: finishing write tid 1 to smithi03385152-35 2024-09-21T09:56:24.618 INFO:tasks.rados.rados.0.smithi033.stdout:1598: finishing write tid 2 to smithi03385152-35 2024-09-21T09:56:24.618 INFO:tasks.rados.rados.0.smithi033.stdout:1598: finishing write tid 3 to smithi03385152-35 2024-09-21T09:56:24.618 INFO:tasks.rados.rados.0.smithi033.stdout:1598: finishing write tid 4 to smithi03385152-35 2024-09-21T09:56:24.618 INFO:tasks.rados.rados.0.smithi033.stdout:1598: finishing write tid 5 to smithi03385152-35 2024-09-21T09:56:24.618 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 993 (ObjNum 523 snap 142 seq_num 523) dirty exists 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:1598: left oid 35 (ObjNum 523 snap 142 seq_num 523) 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:1599: finishing write tid 1 to smithi03385152-29 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:1599: finishing write tid 2 to smithi03385152-29 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:1599: finishing write tid 3 to smithi03385152-29 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:1599: finishing write tid 4 to smithi03385152-29 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:1599: finishing write tid 5 to smithi03385152-29 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 998 (ObjNum 524 snap 142 seq_num 524) dirty exists 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:1599: left oid 29 (ObjNum 524 snap 142 seq_num 524) 2024-09-21T09:56:24.619 INFO:tasks.rados.rados.0.smithi033.stdout:1600: expect (ObjNum 502 snap 132 seq_num 502) 2024-09-21T09:56:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:24 smithi105 ceph-mon[86649]: osdmap e357: 8 total, 7 up, 8 in 2024-09-21T09:56:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:24 smithi105 ceph-mon[86649]: from='osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:56:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:24 smithi105 ceph-mon[86649]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:56:24.938 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 09:56:24 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[108464]: 2024-09-21T09:56:24.527+0000 7f4206517640 -1 osd.1 346 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:56:24.939 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:24 smithi033 ceph-mon[98546]: osdmap e357: 8 total, 7 up, 8 in 2024-09-21T09:56:24.939 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:24 smithi033 ceph-mon[98546]: from='osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:56:24.939 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:24 smithi033 ceph-mon[98546]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:56:24.939 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:24 smithi033 ceph-mon[96568]: osdmap e357: 8 total, 7 up, 8 in 2024-09-21T09:56:24.939 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:24 smithi033 ceph-mon[96568]: from='osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:56:24.939 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:24 smithi033 ceph-mon[96568]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-21T09:56:25.507 INFO:tasks.rados.rados.0.smithi033.stdout:1596: done (5 left) 2024-09-21T09:56:25.507 INFO:tasks.rados.rados.0.smithi033.stdout:1597: done (4 left) 2024-09-21T09:56:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:1598: done (3 left) 2024-09-21T09:56:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:1599: done (2 left) 2024-09-21T09:56:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:1600: done (1 left) 2024-09-21T09:56:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:1601: done (0 left) 2024-09-21T09:56:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:1602: read oid 32 snap -1 2024-09-21T09:56:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:1602: expect deleted 2024-09-21T09:56:25.508 INFO:tasks.rados.rados.0.smithi033.stdout:1603: delete oid 42 current snap is 143 2024-09-21T09:56:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[98546]: pgmap v165: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 150 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 42/273 objects degraded (15.385%) 2024-09-21T09:56:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[98546]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:56:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[98546]: osdmap e358: 8 total, 7 up, 8 in 2024-09-21T09:56:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[98546]: from='osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[98546]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:25.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[96568]: pgmap v165: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 150 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 42/273 objects degraded (15.385%) 2024-09-21T09:56:25.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[96568]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:56:25.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[96568]: osdmap e358: 8 total, 7 up, 8 in 2024-09-21T09:56:25.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[96568]: from='osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:25.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:25 smithi033 ceph-mon[96568]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:25 smithi105 ceph-mon[86649]: pgmap v165: 121 pgs: 32 active+undersized, 12 active+undersized+degraded, 77 active+clean; 150 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 6.2 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 42/273 objects degraded (15.385%) 2024-09-21T09:56:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:25 smithi105 ceph-mon[86649]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-21T09:56:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:25 smithi105 ceph-mon[86649]: osdmap e358: 8 total, 7 up, 8 in 2024-09-21T09:56:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:25 smithi105 ceph-mon[86649]: from='osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:25 smithi105 ceph-mon[86649]: from='osd.1 ' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:25.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:25.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:25.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:26.533 INFO:tasks.rados.rados.0.smithi033.stdout:1602: done (1 left) 2024-09-21T09:56:26.533 INFO:tasks.rados.rados.0.smithi033.stdout:1603: done (0 left) 2024-09-21T09:56:26.533 INFO:tasks.rados.rados.0.smithi033.stdout:1604: read oid 39 snap 139 2024-09-21T09:56:26.533 INFO:tasks.rados.rados.0.smithi033.stdout:1604: expect deleted 2024-09-21T09:56:26.533 INFO:tasks.rados.rados.0.smithi033.stdout:1605: read oid 27 snap 142 2024-09-21T09:56:26.533 INFO:tasks.rados.rados.0.smithi033.stdout:1605: expect (ObjNum 511 snap 136 seq_num 511) 2024-09-21T09:56:26.533 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 143 2024-09-21T09:56:26.533 INFO:tasks.rados.rados.0.smithi033.stdout:1606: seq_num 526 ranges {5447680=417792,5865472=589824,6455296=614400} 2024-09-21T09:56:26.565 INFO:tasks.rados.rados.0.smithi033.stdout:1606: writing smithi03385152-5 from 5447680 to 5865472 tid 1 2024-09-21T09:56:26.568 INFO:tasks.rados.rados.0.smithi033.stdout:1606: writing smithi03385152-5 from 5865472 to 6455296 tid 2 2024-09-21T09:56:26.572 INFO:tasks.rados.rados.0.smithi033.stdout:1606: writing smithi03385152-5 from 6455296 to 7069696 tid 3 2024-09-21T09:56:26.572 INFO:tasks.rados.rados.0.smithi033.stdout:1607: rollback oid 38 current snap is 143 2024-09-21T09:56:26.572 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 134 2024-09-21T09:56:26.572 INFO:tasks.rados.rados.0.smithi033.stdout:1605: expect (ObjNum 511 snap 136 seq_num 511) 2024-09-21T09:56:26.616 INFO:tasks.rados.rados.0.smithi033.stdout:1606: finishing write tid 1 to smithi03385152-5 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1606: finishing write tid 2 to smithi03385152-5 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1606: finishing write tid 3 to smithi03385152-5 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1606: finishing write tid 4 to smithi03385152-5 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1607: finishing rollback tid 0 to smithi03385152-38 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1606: finishing write tid 5 to smithi03385152-5 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 223 (ObjNum 526 snap 143 seq_num 526) dirty exists 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1606: left oid 5 (ObjNum 526 snap 143 seq_num 526) 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1607: finishing rollback tid 1 to smithi03385152-38 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1607: finishing rollback tid 2 to smithi03385152-38 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 727 (ObjNum 499 snap 128 seq_num 499) dirty exists 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1604: done (3 left) 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1605: done (2 left) 2024-09-21T09:56:26.617 INFO:tasks.rados.rados.0.smithi033.stdout:1606: done (1 left) 2024-09-21T09:56:26.618 INFO:tasks.rados.rados.0.smithi033.stdout:1607: done (0 left) 2024-09-21T09:56:26.618 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 143 2024-09-21T09:56:26.618 INFO:tasks.rados.rados.0.smithi033.stdout:1608: seq_num 527 ranges {0=434176,434176=475136,909312=565248,1474560=163840} 2024-09-21T09:56:26.618 INFO:tasks.rados.rados.0.smithi033.stdout:1608: writing smithi03385152-30 from 0 to 434176 tid 1 2024-09-21T09:56:26.621 INFO:tasks.rados.rados.0.smithi033.stdout:1608: writing smithi03385152-30 from 434176 to 909312 tid 2 2024-09-21T09:56:26.624 INFO:tasks.rados.rados.0.smithi033.stdout:1608: writing smithi03385152-30 from 909312 to 1474560 tid 3 2024-09-21T09:56:26.626 INFO:tasks.rados.rados.0.smithi033.stdout:1608: writing smithi03385152-30 from 1474560 to 1638400 tid 4 2024-09-21T09:56:26.626 INFO:tasks.rados.rados.0.smithi033.stdout:1609: setattr oid 38 current snap is 143 2024-09-21T09:56:26.626 INFO:tasks.rados.rados.0.smithi033.stdout:1608: finishing write tid 1 to smithi03385152-30 2024-09-21T09:56:26.626 INFO:tasks.rados.rados.0.smithi033.stdout:1608: finishing write tid 2 to smithi03385152-30 2024-09-21T09:56:26.628 INFO:tasks.rados.rados.0.smithi033.stdout:1610: copy_from oid 16 from oid 17 current snap is 143 2024-09-21T09:56:26.628 INFO:tasks.rados.rados.0.smithi033.stdout:1611: read oid 9 snap -1 2024-09-21T09:56:26.628 INFO:tasks.rados.rados.0.smithi033.stdout:1611: expect (ObjNum 479 snap 125 seq_num 479) 2024-09-21T09:56:26.628 INFO:tasks.rados.rados.0.smithi033.stdout:1612: snap_remove snap 131 2024-09-21T09:56:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[98546]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:56:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[98546]: osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800] boot 2024-09-21T09:56:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[98546]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:56:26.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:56:26.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[98546]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:56:26.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[96568]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:56:26.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[96568]: osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800] boot 2024-09-21T09:56:26.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[96568]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:56:26.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:56:26.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-mon[96568]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:56:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:26 smithi105 ceph-mon[86649]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:56:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:26 smithi105 ceph-mon[86649]: osd.1 [v2:172.21.15.33:6810/2766504800,v1:172.21.15.33:6811/2766504800] boot 2024-09-21T09:56:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:26 smithi105 ceph-mon[86649]: osdmap e359: 8 total, 8 up, 8 in 2024-09-21T09:56:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-21T09:56:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:26 smithi105 ceph-mon[86649]: osdmap e360: 8 total, 8 up, 8 in 2024-09-21T09:56:27.246 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:56:26 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:56:26.903+0000 7fc38e955640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:56:27.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:27.536 INFO:tasks.rados.rados.0.smithi033.stdout:1612: done (4 left) 2024-09-21T09:56:27.536 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 143 2024-09-21T09:56:27.536 INFO:tasks.rados.rados.0.smithi033.stdout:1613: seq_num 528 ranges {0=450560,450560=761856,1212416=442368} 2024-09-21T09:56:27.538 INFO:tasks.rados.rados.0.smithi033.stdout:1613: writing smithi03385152-19 from 0 to 450560 tid 1 2024-09-21T09:56:27.543 INFO:tasks.rados.rados.0.smithi033.stdout:1613: writing smithi03385152-19 from 450560 to 1212416 tid 2 2024-09-21T09:56:27.546 INFO:tasks.rados.rados.0.smithi033.stdout:1613: writing smithi03385152-19 from 1212416 to 1654784 tid 3 2024-09-21T09:56:27.546 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 143 2024-09-21T09:56:27.547 INFO:tasks.rados.rados.0.smithi033.stdout:1608: finishing write tid 3 to smithi03385152-30 2024-09-21T09:56:27.547 INFO:tasks.rados.rados.0.smithi033.stdout:1608: finishing write tid 4 to smithi03385152-30 2024-09-21T09:56:27.547 INFO:tasks.rados.rados.0.smithi033.stdout:1608: finishing write tid 5 to smithi03385152-30 2024-09-21T09:56:27.547 INFO:tasks.rados.rados.0.smithi033.stdout:1608: finishing write tid 6 to smithi03385152-30 2024-09-21T09:56:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 1013 (ObjNum 527 snap 143 seq_num 527) dirty exists 2024-09-21T09:56:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:1608: left oid 30 (ObjNum 527 snap 143 seq_num 527) 2024-09-21T09:56:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 728 (ObjNum 499 snap 128 seq_num 499) dirty exists 2024-09-21T09:56:27.548 INFO:tasks.rados.rados.0.smithi033.stdout:1611: expect (ObjNum 479 snap 125 seq_num 479) 2024-09-21T09:56:27.601 INFO:tasks.rados.rados.0.smithi033.stdout:1610: finishing copy_from racing read to smithi03385152-16 2024-09-21T09:56:27.601 INFO:tasks.rados.rados.0.smithi033.stdout:1610: finishing copy_from to smithi03385152-16 2024-09-21T09:56:27.601 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 553 (ObjNum 507 snap 135 seq_num 507) dirty exists 2024-09-21T09:56:27.601 INFO:tasks.rados.rados.0.smithi033.stdout:1613: finishing write tid 1 to smithi03385152-19 2024-09-21T09:56:27.601 INFO:tasks.rados.rados.0.smithi033.stdout:1614: seq_num 529 ranges {7733248=466944,8200192=647168,8847360=532480,9379840=24576} 2024-09-21T09:56:27.645 INFO:tasks.rados.rados.0.smithi033.stdout:1614: writing smithi03385152-22 from 7733248 to 8200192 tid 1 2024-09-21T09:56:27.649 INFO:tasks.rados.rados.0.smithi033.stdout:1614: writing smithi03385152-22 from 8200192 to 8847360 tid 2 2024-09-21T09:56:27.652 INFO:tasks.rados.rados.0.smithi033.stdout:1614: writing smithi03385152-22 from 8847360 to 9379840 tid 3 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1614: writing smithi03385152-22 from 9379840 to 9404416 tid 4 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1608: done (5 left) 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1609: done (4 left) 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1610: done (3 left) 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1611: done (2 left) 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1615: copy_from oid 44 from oid 29 current snap is 143 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1613: finishing write tid 2 to smithi03385152-19 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1613: finishing write tid 3 to smithi03385152-19 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1613: finishing write tid 4 to smithi03385152-19 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1613: finishing write tid 5 to smithi03385152-19 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 666 (ObjNum 528 snap 143 seq_num 528) dirty exists 2024-09-21T09:56:27.653 INFO:tasks.rados.rados.0.smithi033.stdout:1613: left oid 19 (ObjNum 528 snap 143 seq_num 528) 2024-09-21T09:56:27.654 INFO:tasks.rados.rados.0.smithi033.stdout:1614: finishing write tid 1 to smithi03385152-22 2024-09-21T09:56:27.654 INFO:tasks.rados.rados.0.smithi033.stdout:1613: done (2 left) 2024-09-21T09:56:27.654 INFO:tasks.rados.rados.0.smithi033.stdout:1616: snap_remove snap 142 2024-09-21T09:56:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: pgmap v168: 121 pgs: 28 peering, 11 active+undersized, 5 active+undersized+degraded, 77 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.2 MiB/s wr, 14 op/s; 8/294 objects degraded (2.721%) 2024-09-21T09:56:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[96568]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: pgmap v168: 121 pgs: 28 peering, 11 active+undersized, 5 active+undersized+degraded, 77 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.2 MiB/s wr, 14 op/s; 8/294 objects degraded (2.721%) 2024-09-21T09:56:28.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:56:28.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:27 smithi033 ceph-mon[98546]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:56:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: pgmap v168: 121 pgs: 28 peering, 11 active+undersized, 5 active+undersized+degraded, 77 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.2 MiB/s wr, 14 op/s; 8/294 objects degraded (2.721%) 2024-09-21T09:56:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: Upgrade: unsafe to stop osd(s) at this time (11 PGs are or would become offline) 2024-09-21T09:56:28.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:27 smithi105 ceph-mon[86649]: osdmap e361: 8 total, 8 up, 8 in 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1616: done (2 left) 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1617: read oid 50 snap -1 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1617: expect deleted 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1618: read oid 7 snap 134 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1618: expect (ObjNum 474 snap 125 seq_num 474) 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1619: delete oid 14 current snap is 143 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1614: finishing write tid 2 to smithi03385152-22 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1614: finishing write tid 3 to smithi03385152-22 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1614: finishing write tid 4 to smithi03385152-22 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1614: finishing write tid 5 to smithi03385152-22 2024-09-21T09:56:28.535 INFO:tasks.rados.rados.0.smithi033.stdout:1614: finishing write tid 6 to smithi03385152-22 2024-09-21T09:56:28.536 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 879 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:56:28.536 INFO:tasks.rados.rados.0.smithi033.stdout:1614: left oid 22 (ObjNum 529 snap 143 seq_num 529) 2024-09-21T09:56:28.536 INFO:tasks.rados.rados.0.smithi033.stdout:1615: finishing copy_from racing read to smithi03385152-44 2024-09-21T09:56:28.536 INFO:tasks.rados.rados.0.smithi033.stdout:1615: finishing copy_from to smithi03385152-44 2024-09-21T09:56:28.536 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 998 (ObjNum 524 snap 142 seq_num 524) dirty exists 2024-09-21T09:56:28.542 INFO:tasks.rados.rados.0.smithi033.stdout:1614: done (4 left) 2024-09-21T09:56:28.542 INFO:tasks.rados.rados.0.smithi033.stdout:1615: done (3 left) 2024-09-21T09:56:28.542 INFO:tasks.rados.rados.0.smithi033.stdout:1617: done (2 left) 2024-09-21T09:56:28.542 INFO:tasks.rados.rados.0.smithi033.stdout:1619: done (1 left) 2024-09-21T09:56:28.542 INFO:tasks.rados.rados.0.smithi033.stdout:1620: snap_remove snap 139 2024-09-21T09:56:29.181 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:28 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 8/294 objects degraded (2.721%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:29.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:28 smithi105 ceph-mon[86649]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:56:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:28 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 8/294 objects degraded (2.721%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:28 smithi033 ceph-mon[98546]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:56:29.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:28 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 8/294 objects degraded (2.721%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:29.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:28 smithi033 ceph-mon[96568]: osdmap e362: 8 total, 8 up, 8 in 2024-09-21T09:56:29.534 INFO:tasks.rados.rados.0.smithi033.stdout:1620: done (1 left) 2024-09-21T09:56:29.535 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 143 2024-09-21T09:56:29.535 INFO:tasks.rados.rados.0.smithi033.stdout:1621: seq_num 531 ranges {0=499712,499712=417792,917504=548864,1466368=237568} 2024-09-21T09:56:29.537 INFO:tasks.rados.rados.0.smithi033.stdout:1621: writing smithi03385152-33 from 0 to 499712 tid 1 2024-09-21T09:56:29.540 INFO:tasks.rados.rados.0.smithi033.stdout:1621: writing smithi03385152-33 from 499712 to 917504 tid 2 2024-09-21T09:56:29.544 INFO:tasks.rados.rados.0.smithi033.stdout:1621: writing smithi03385152-33 from 917504 to 1466368 tid 3 2024-09-21T09:56:29.546 INFO:tasks.rados.rados.0.smithi033.stdout:1621: writing smithi03385152-33 from 1466368 to 1703936 tid 4 2024-09-21T09:56:29.546 INFO:tasks.rados.rados.0.smithi033.stdout:1622: setattr oid 31 current snap is 143 2024-09-21T09:56:29.546 INFO:tasks.rados.rados.0.smithi033.stdout:1618: expect (ObjNum 474 snap 125 seq_num 474) 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1621: finishing write tid 1 to smithi03385152-33 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1621: finishing write tid 2 to smithi03385152-33 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1621: finishing write tid 3 to smithi03385152-33 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1621: finishing write tid 4 to smithi03385152-33 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1621: finishing write tid 5 to smithi03385152-33 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1621: finishing write tid 6 to smithi03385152-33 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 845 (ObjNum 531 snap 143 seq_num 531) dirty exists 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1621: left oid 33 (ObjNum 531 snap 143 seq_num 531) 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1618: done (2 left) 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:1621: done (1 left) 2024-09-21T09:56:29.570 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 143 2024-09-21T09:56:29.571 INFO:tasks.rados.rados.0.smithi033.stdout:1623: seq_num 532 ranges {1114112=516096,1630208=696320,2326528=507904} 2024-09-21T09:56:29.578 INFO:tasks.rados.rados.0.smithi033.stdout:1623: writing smithi03385152-20 from 1114112 to 1630208 tid 1 2024-09-21T09:56:29.582 INFO:tasks.rados.rados.0.smithi033.stdout:1623: writing smithi03385152-20 from 1630208 to 2326528 tid 2 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:1623: writing smithi03385152-20 from 2326528 to 2834432 tid 3 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:1624: copy_from oid 45 from oid 28 current snap is 143 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1029 (ObjNum 519 snap 142 seq_num 519) dirty exists 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:1623: finishing write tid 1 to smithi03385152-20 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:1622: done (2 left) 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:1625: rollback oid 3 current snap is 143 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 3 to 134 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:1626: delete oid 29 current snap is 143 2024-09-21T09:56:29.586 INFO:tasks.rados.rados.0.smithi033.stdout:1623: finishing write tid 2 to smithi03385152-20 2024-09-21T09:56:29.588 INFO:tasks.rados.rados.0.smithi033.stdout:1624: finishing copy_from to smithi03385152-45 2024-09-21T09:56:29.588 INFO:tasks.rados.rados.0.smithi033.stdout:1624: got expected ENOENT (src dne) 2024-09-21T09:56:29.588 INFO:tasks.rados.rados.0.smithi033.stdout:1623: finishing write tid 3 to smithi03385152-20 2024-09-21T09:56:29.588 INFO:tasks.rados.rados.0.smithi033.stdout:1623: finishing write tid 4 to smithi03385152-20 2024-09-21T09:56:29.588 INFO:tasks.rados.rados.0.smithi033.stdout:1624: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:56:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:1623: finishing write tid 5 to smithi03385152-20 2024-09-21T09:56:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 702 (ObjNum 532 snap 143 seq_num 532) dirty exists 2024-09-21T09:56:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:1623: left oid 20 (ObjNum 532 snap 143 seq_num 532) 2024-09-21T09:56:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:1625: finishing rollback tid 0 to smithi03385152-3 2024-09-21T09:56:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:1625: finishing rollback tid 1 to smithi03385152-3 2024-09-21T09:56:29.590 INFO:tasks.rados.rados.0.smithi033.stdout:1625: finishing rollback tid 2 to smithi03385152-3 2024-09-21T09:56:29.590 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 891 (ObjNum 450 snap 120 seq_num 450) dirty exists 2024-09-21T09:56:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:1623: done (3 left) 2024-09-21T09:56:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:1624: done (2 left) 2024-09-21T09:56:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:1625: done (1 left) 2024-09-21T09:56:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:1626: done (0 left) 2024-09-21T09:56:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:1627: copy_from oid 37 from oid 47 current snap is 143 2024-09-21T09:56:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:1628: copy_from oid 17 from oid 36 current snap is 143 2024-09-21T09:56:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:1629: read oid 27 snap -1 2024-09-21T09:56:29.591 INFO:tasks.rados.rados.0.smithi033.stdout:1629: expect (ObjNum 511 snap 136 seq_num 511) 2024-09-21T09:56:29.592 INFO:tasks.rados.rados.0.smithi033.stdout:1630: delete oid 23 current snap is 143 2024-09-21T09:56:29.592 INFO:tasks.rados.rados.0.smithi033.stdout:1628: finishing copy_from to smithi03385152-17 2024-09-21T09:56:29.592 INFO:tasks.rados.rados.0.smithi033.stdout:1628: got expected ENOENT (src dne) 2024-09-21T09:56:29.592 INFO:tasks.rados.rados.0.smithi033.stdout:1628: finishing copy_from racing read to smithi03385152-17 2024-09-21T09:56:29.594 INFO:tasks.rados.rados.0.smithi033.stdout:1628: done (3 left) 2024-09-21T09:56:29.594 INFO:tasks.rados.rados.0.smithi033.stdout:1630: done (2 left) 2024-09-21T09:56:29.594 INFO:tasks.rados.rados.0.smithi033.stdout:1631: rmattr oid 26 current snap is 143 2024-09-21T09:56:29.594 INFO:tasks.rados.rados.0.smithi033.stdout:1632: read oid 46 snap -1 2024-09-21T09:56:29.594 INFO:tasks.rados.rados.0.smithi033.stdout:1632: expect deleted 2024-09-21T09:56:29.594 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 143 2024-09-21T09:56:29.594 INFO:tasks.rados.rados.0.smithi033.stdout:1633: seq_num 535 ranges {860160=565248,1425408=802816,2228224=401408} 2024-09-21T09:56:29.602 INFO:tasks.rados.rados.0.smithi033.stdout:1633: writing smithi03385152-25 from 860160 to 1425408 tid 1 2024-09-21T09:56:29.607 INFO:tasks.rados.rados.0.smithi033.stdout:1633: writing smithi03385152-25 from 1425408 to 2228224 tid 2 2024-09-21T09:56:29.610 INFO:tasks.rados.rados.0.smithi033.stdout:1633: writing smithi03385152-25 from 2228224 to 2629632 tid 3 2024-09-21T09:56:29.610 INFO:tasks.rados.rados.0.smithi033.stdout:1634: delete oid 42 current snap is 143 2024-09-21T09:56:29.610 INFO:tasks.rados.rados.0.smithi033.stdout:1629: expect (ObjNum 511 snap 136 seq_num 511) 2024-09-21T09:56:29.652 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 840 (ObjNum 345 snap 91 seq_num 345) dirty exists 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1633: finishing write tid 1 to smithi03385152-25 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1633: finishing write tid 2 to smithi03385152-25 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1633: finishing write tid 3 to smithi03385152-25 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1633: finishing write tid 4 to smithi03385152-25 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1633: finishing write tid 5 to smithi03385152-25 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 650 (ObjNum 535 snap 143 seq_num 535) dirty exists 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1633: left oid 25 (ObjNum 535 snap 143 seq_num 535) 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1627: finishing copy_from racing read to smithi03385152-37 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1627: finishing copy_from to smithi03385152-37 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 650 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1627: done (5 left) 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1629: done (4 left) 2024-09-21T09:56:29.653 INFO:tasks.rados.rados.0.smithi033.stdout:1631: done (3 left) 2024-09-21T09:56:29.654 INFO:tasks.rados.rados.0.smithi033.stdout:1632: done (2 left) 2024-09-21T09:56:29.654 INFO:tasks.rados.rados.0.smithi033.stdout:1633: done (1 left) 2024-09-21T09:56:29.654 INFO:tasks.rados.rados.0.smithi033.stdout:1634: done (0 left) 2024-09-21T09:56:29.654 INFO:tasks.rados.rados.0.smithi033.stdout:1635: snap_remove snap 134 2024-09-21T09:56:30.201 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:29 smithi105 ceph-mon[86649]: pgmap v171: 121 pgs: 28 peering, 11 active+undersized, 5 active+undersized+degraded, 77 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.2 MiB/s wr, 14 op/s; 8/294 objects degraded (2.721%) 2024-09-21T09:56:30.201 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:29 smithi105 ceph-mon[86649]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:56:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:29 smithi033 ceph-mon[98546]: pgmap v171: 121 pgs: 28 peering, 11 active+undersized, 5 active+undersized+degraded, 77 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.2 MiB/s wr, 14 op/s; 8/294 objects degraded (2.721%) 2024-09-21T09:56:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:29 smithi033 ceph-mon[98546]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:56:30.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:29 smithi033 ceph-mon[96568]: pgmap v171: 121 pgs: 28 peering, 11 active+undersized, 5 active+undersized+degraded, 77 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 2.2 MiB/s wr, 14 op/s; 8/294 objects degraded (2.721%) 2024-09-21T09:56:30.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:29 smithi033 ceph-mon[96568]: osdmap e363: 8 total, 8 up, 8 in 2024-09-21T09:56:30.454 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:1635: done (0 left) 2024-09-21T09:56:30.537 INFO:tasks.rados.rados.0.smithi033.stdout:1636: delete oid 24 current snap is 143 2024-09-21T09:56:30.547 INFO:tasks.rados.rados.0.smithi033.stdout:1636: done (0 left) 2024-09-21T09:56:30.547 INFO:tasks.rados.rados.0.smithi033.stdout:1637: delete oid 23 current snap is 143 2024-09-21T09:56:30.555 INFO:tasks.rados.rados.0.smithi033.stdout:1637: done (0 left) 2024-09-21T09:56:30.555 INFO:tasks.rados.rados.0.smithi033.stdout:1638: copy_from oid 6 from oid 30 current snap is 143 2024-09-21T09:56:30.555 INFO:tasks.rados.rados.0.smithi033.stdout:1639: snap_create 2024-09-21T09:56:30.563 INFO:tasks.rados.rados.0.smithi033.stdout:1638: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:56:30.564 INFO:tasks.rados.rados.0.smithi033.stdout:1638: finishing copy_from to smithi03385152-6 2024-09-21T09:56:30.564 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1013 (ObjNum 527 snap 143 seq_num 527) dirty exists 2024-09-21T09:56:31.543 INFO:tasks.rados.rados.0.smithi033.stdout:1638: done (1 left) 2024-09-21T09:56:31.570 INFO:tasks.rados.rados.0.smithi033.stdout:1639: done (0 left) 2024-09-21T09:56:31.571 INFO:tasks.rados.rados.0.smithi033.stdout:1640: delete oid 46 current snap is 144 2024-09-21T09:56:31.573 INFO:tasks.rados.rados.0.smithi033.stdout:1640: done (0 left) 2024-09-21T09:56:31.573 INFO:tasks.rados.rados.0.smithi033.stdout:1641: rollback oid 6 current snap is 144 2024-09-21T09:56:31.574 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 143 2024-09-21T09:56:31.574 INFO:tasks.rados.rados.0.smithi033.stdout:1642: read oid 17 snap -1 2024-09-21T09:56:31.574 INFO:tasks.rados.rados.0.smithi033.stdout:1642: expect (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:31.574 INFO:tasks.rados.rados.0.smithi033.stdout:1643: rmattr oid 26 current snap is 144 2024-09-21T09:56:31.574 INFO:tasks.rados.rados.0.smithi033.stdout:1644: snap_create 2024-09-21T09:56:31.574 INFO:tasks.rados.rados.0.smithi033.stdout:1641: finishing rollback tid 0 to smithi03385152-6 2024-09-21T09:56:31.575 INFO:tasks.rados.rados.0.smithi033.stdout:1641: finishing rollback tid 1 to smithi03385152-6 2024-09-21T09:56:31.575 INFO:tasks.rados.rados.0.smithi033.stdout:1641: finishing rollback tid 2 to smithi03385152-6 2024-09-21T09:56:31.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1016 (ObjNum 527 snap 143 seq_num 527) dirty exists 2024-09-21T09:56:31.581 INFO:tasks.rados.rados.0.smithi033.stdout:1642: expect (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:31 smithi105 ceph-mon[86649]: pgmap v174: 121 pgs: 28 peering, 93 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:31 smithi105 ceph-mon[86649]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:56:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:31 smithi105 ceph-mon[86649]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/294 objects degraded (2.721%), 5 pgs degraded) 2024-09-21T09:56:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:31 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T09:56:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:56:31] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.51.0" 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[96568]: pgmap v174: 121 pgs: 28 peering, 93 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[96568]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[96568]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/294 objects degraded (2.721%), 5 pgs degraded) 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[98546]: pgmap v174: 121 pgs: 28 peering, 93 active+clean; 157 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[98546]: osdmap e364: 8 total, 8 up, 8 in 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:31.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[98546]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/294 objects degraded (2.721%), 5 pgs degraded) 2024-09-21T09:56:31.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:31 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T09:56:32.145 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 841 (ObjNum 345 snap 91 seq_num 345) dirty exists 2024-09-21T09:56:32.546 INFO:tasks.rados.rados.0.smithi033.stdout:1641: done (3 left) 2024-09-21T09:56:32.547 INFO:tasks.rados.rados.0.smithi033.stdout:1642: done (2 left) 2024-09-21T09:56:32.547 INFO:tasks.rados.rados.0.smithi033.stdout:1643: done (1 left) 2024-09-21T09:56:32.547 INFO:tasks.rados.rados.0.smithi033.stdout:1644: done (0 left) 2024-09-21T09:56:32.547 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 145 2024-09-21T09:56:32.547 INFO:tasks.rados.rados.0.smithi033.stdout:1645: seq_num 540 ranges {0=647168,647168=671744,1318912=532480} 2024-09-21T09:56:32.549 INFO:tasks.rados.rados.0.smithi033.stdout:1645: writing smithi03385152-32 from 0 to 647168 tid 1 2024-09-21T09:56:32.553 INFO:tasks.rados.rados.0.smithi033.stdout:1645: writing smithi03385152-32 from 647168 to 1318912 tid 2 2024-09-21T09:56:32.557 INFO:tasks.rados.rados.0.smithi033.stdout:1645: writing smithi03385152-32 from 1318912 to 1851392 tid 3 2024-09-21T09:56:32.557 INFO:tasks.rados.rados.0.smithi033.stdout:1646: rollback oid 4 current snap is 145 2024-09-21T09:56:32.558 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 4 to 143 2024-09-21T09:56:32.558 INFO:tasks.rados.rados.0.smithi033.stdout:1645: finishing write tid 1 to smithi03385152-32 2024-09-21T09:56:32.558 INFO:tasks.rados.rados.0.smithi033.stdout:1645: finishing write tid 2 to smithi03385152-32 2024-09-21T09:56:32.558 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 145 2024-09-21T09:56:32.558 INFO:tasks.rados.rados.0.smithi033.stdout:1647: seq_num 541 ranges {7503872=663552,8167424=557056,8724480=475136,9199616=172032} 2024-09-21T09:56:32.602 INFO:tasks.rados.rados.0.smithi033.stdout:1647: writing smithi03385152-17 from 7503872 to 8167424 tid 1 2024-09-21T09:56:32.605 INFO:tasks.rados.rados.0.smithi033.stdout:1647: writing smithi03385152-17 from 8167424 to 8724480 tid 2 2024-09-21T09:56:32.608 INFO:tasks.rados.rados.0.smithi033.stdout:1647: writing smithi03385152-17 from 8724480 to 9199616 tid 3 2024-09-21T09:56:32.609 INFO:tasks.rados.rados.0.smithi033.stdout:1647: writing smithi03385152-17 from 9199616 to 9371648 tid 4 2024-09-21T09:56:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:1648: rollback oid 34 current snap is 145 2024-09-21T09:56:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 144 2024-09-21T09:56:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:1645: finishing write tid 3 to smithi03385152-32 2024-09-21T09:56:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:1645: finishing write tid 4 to smithi03385152-32 2024-09-21T09:56:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:1645: finishing write tid 5 to smithi03385152-32 2024-09-21T09:56:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 166 (ObjNum 540 snap 145 seq_num 540) dirty exists 2024-09-21T09:56:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:1645: left oid 32 (ObjNum 540 snap 145 seq_num 540) 2024-09-21T09:56:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:1646: finishing rollback tid 0 to smithi03385152-4 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:1646: finishing rollback tid 1 to smithi03385152-4 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:1646: finishing rollback tid 2 to smithi03385152-4 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1001 (ObjNum 502 snap 132 seq_num 502) dirty exists 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:1647: finishing write tid 1 to smithi03385152-17 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:1645: done (3 left) 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:1646: done (2 left) 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:1649: setattr oid 16 current snap is 145 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:1647: finishing write tid 2 to smithi03385152-17 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:1648: finishing rollback tid 1 to smithi03385152-34 2024-09-21T09:56:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:56:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:1648: done (2 left) 2024-09-21T09:56:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 145 2024-09-21T09:56:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:1650: seq_num 542 ranges {4874240=679936,5554176=442368,5996544=679936,6676480=81920} 2024-09-21T09:56:32.641 INFO:tasks.rados.rados.0.smithi033.stdout:1650: writing smithi03385152-15 from 4874240 to 5554176 tid 1 2024-09-21T09:56:32.643 INFO:tasks.rados.rados.0.smithi033.stdout:1650: writing smithi03385152-15 from 5554176 to 5996544 tid 2 2024-09-21T09:56:32.647 INFO:tasks.rados.rados.0.smithi033.stdout:1650: writing smithi03385152-15 from 5996544 to 6676480 tid 3 2024-09-21T09:56:32.648 INFO:tasks.rados.rados.0.smithi033.stdout:1650: writing smithi03385152-15 from 6676480 to 6758400 tid 4 2024-09-21T09:56:32.648 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 145 2024-09-21T09:56:32.648 INFO:tasks.rados.rados.0.smithi033.stdout:1651: seq_num 543 ranges {0=704512,704512=729088,1433600=475136} 2024-09-21T09:56:32.652 INFO:tasks.rados.rados.0.smithi033.stdout:1651: writing smithi03385152-42 from 0 to 704512 tid 1 2024-09-21T09:56:32.656 INFO:tasks.rados.rados.0.smithi033.stdout:1651: writing smithi03385152-42 from 704512 to 1433600 tid 2 2024-09-21T09:56:32.659 INFO:tasks.rados.rados.0.smithi033.stdout:1651: writing smithi03385152-42 from 1433600 to 1908736 tid 3 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1652: read oid 12 snap -1 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1652: expect (ObjNum 480 snap 125 seq_num 480) 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1653: read oid 43 snap -1 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1653: expect (ObjNum 510 snap 135 seq_num 510) 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1654: snap_create 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1647: finishing write tid 3 to smithi03385152-17 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1647: finishing write tid 4 to smithi03385152-17 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1647: finishing write tid 5 to smithi03385152-17 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1647: finishing write tid 6 to smithi03385152-17 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 846 (ObjNum 541 snap 145 seq_num 541) dirty exists 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1647: left oid 17 (ObjNum 541 snap 145 seq_num 541) 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 1038 (ObjNum 507 snap 135 seq_num 507) dirty exists 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1650: finishing write tid 1 to smithi03385152-15 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1650: finishing write tid 2 to smithi03385152-15 2024-09-21T09:56:32.660 INFO:tasks.rados.rados.0.smithi033.stdout:1650: finishing write tid 3 to smithi03385152-15 2024-09-21T09:56:32.661 INFO:tasks.rados.rados.0.smithi033.stdout:1650: finishing write tid 4 to smithi03385152-15 2024-09-21T09:56:32.661 INFO:tasks.rados.rados.0.smithi033.stdout:1650: finishing write tid 5 to smithi03385152-15 2024-09-21T09:56:32.661 INFO:tasks.rados.rados.0.smithi033.stdout:1650: finishing write tid 6 to smithi03385152-15 2024-09-21T09:56:32.661 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 665 (ObjNum 542 snap 145 seq_num 542) dirty exists 2024-09-21T09:56:32.661 INFO:tasks.rados.rados.0.smithi033.stdout:1650: left oid 15 (ObjNum 542 snap 145 seq_num 542) 2024-09-21T09:56:32.661 INFO:tasks.rados.rados.0.smithi033.stdout:1651: finishing write tid 1 to smithi03385152-42 2024-09-21T09:56:32.661 INFO:tasks.rados.rados.0.smithi033.stdout:1651: finishing write tid 2 to smithi03385152-42 2024-09-21T09:56:32.662 INFO:tasks.rados.rados.0.smithi033.stdout:1651: finishing write tid 3 to smithi03385152-42 2024-09-21T09:56:32.662 INFO:tasks.rados.rados.0.smithi033.stdout:1651: finishing write tid 4 to smithi03385152-42 2024-09-21T09:56:32.664 INFO:tasks.rados.rados.0.smithi033.stdout:1651: finishing write tid 5 to smithi03385152-42 2024-09-21T09:56:32.664 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 732 (ObjNum 543 snap 145 seq_num 543) dirty exists 2024-09-21T09:56:32.664 INFO:tasks.rados.rados.0.smithi033.stdout:1651: left oid 42 (ObjNum 543 snap 145 seq_num 543) 2024-09-21T09:56:32.677 INFO:tasks.rados.rados.0.smithi033.stdout:1652: expect (ObjNum 480 snap 125 seq_num 480) 2024-09-21T09:56:32.789 INFO:tasks.rados.rados.0.smithi033.stdout:1653: expect (ObjNum 510 snap 135 seq_num 510) 2024-09-21T09:56:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:32 smithi033 ceph-mon[96568]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:56:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:32 smithi033 ceph-mon[98546]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:56:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:32 smithi105 ceph-mon[86649]: osdmap e365: 8 total, 8 up, 8 in 2024-09-21T09:56:33.456 INFO:tasks.rados.rados.0.smithi033.stdout:1647: done (6 left) 2024-09-21T09:56:33.456 INFO:tasks.rados.rados.0.smithi033.stdout:1649: done (5 left) 2024-09-21T09:56:33.456 INFO:tasks.rados.rados.0.smithi033.stdout:1650: done (4 left) 2024-09-21T09:56:33.456 INFO:tasks.rados.rados.0.smithi033.stdout:1651: done (3 left) 2024-09-21T09:56:33.457 INFO:tasks.rados.rados.0.smithi033.stdout:1652: done (2 left) 2024-09-21T09:56:33.457 INFO:tasks.rados.rados.0.smithi033.stdout:1653: done (1 left) 2024-09-21T09:56:33.457 INFO:tasks.rados.rados.0.smithi033.stdout:1654: done (0 left) 2024-09-21T09:56:33.457 INFO:tasks.rados.rados.0.smithi033.stdout:1655: read oid 24 snap -1 2024-09-21T09:56:33.457 INFO:tasks.rados.rados.0.smithi033.stdout:1655: expect deleted 2024-09-21T09:56:33.457 INFO:tasks.rados.rados.0.smithi033.stdout:1656: read oid 14 snap -1 2024-09-21T09:56:33.457 INFO:tasks.rados.rados.0.smithi033.stdout:1656: expect deleted 2024-09-21T09:56:33.457 INFO:tasks.rados.rados.0.smithi033.stdout:1657: snap_remove snap 145 2024-09-21T09:56:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:33 smithi033 ceph-mon[98546]: pgmap v177: 121 pgs: 121 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.8 MiB/s wr, 32 op/s; 8.4 MiB/s, 4 objects/s recovering 2024-09-21T09:56:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:33 smithi033 ceph-mon[98546]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:56:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:33 smithi033 ceph-mon[98546]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:56:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:33 smithi033 ceph-mon[96568]: pgmap v177: 121 pgs: 121 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.8 MiB/s wr, 32 op/s; 8.4 MiB/s, 4 objects/s recovering 2024-09-21T09:56:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:33 smithi033 ceph-mon[96568]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:56:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:33 smithi033 ceph-mon[96568]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:56:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:33 smithi105 ceph-mon[86649]: pgmap v177: 121 pgs: 121 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.8 MiB/s wr, 32 op/s; 8.4 MiB/s, 4 objects/s recovering 2024-09-21T09:56:34.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:33 smithi105 ceph-mon[86649]: osdmap e366: 8 total, 8 up, 8 in 2024-09-21T09:56:34.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:33 smithi105 ceph-mon[86649]: osdmap e367: 8 total, 8 up, 8 in 2024-09-21T09:56:34.459 INFO:tasks.rados.rados.0.smithi033.stdout:1657: done (2 left) 2024-09-21T09:56:34.459 INFO:tasks.rados.rados.0.smithi033.stdout:1658: snap_remove snap 144 2024-09-21T09:56:34.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:34.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:35.463 INFO:tasks.rados.rados.0.smithi033.stdout:1658: done (2 left) 2024-09-21T09:56:35.465 INFO:tasks.rados.rados.0.smithi033.stdout:1659: snap_create 2024-09-21T09:56:35.743 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:35 smithi105 ceph-mon[86649]: pgmap v180: 121 pgs: 121 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.8 MiB/s wr, 32 op/s; 8.4 MiB/s, 4 objects/s recovering 2024-09-21T09:56:35.744 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:35 smithi105 ceph-mon[86649]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:56:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:35 smithi033 ceph-mon[98546]: pgmap v180: 121 pgs: 121 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.8 MiB/s wr, 32 op/s; 8.4 MiB/s, 4 objects/s recovering 2024-09-21T09:56:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:35 smithi033 ceph-mon[98546]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:56:35.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:35 smithi033 ceph-mon[96568]: pgmap v180: 121 pgs: 121 active+clean; 121 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.8 MiB/s wr, 32 op/s; 8.4 MiB/s, 4 objects/s recovering 2024-09-21T09:56:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:35 smithi033 ceph-mon[96568]: osdmap e368: 8 total, 8 up, 8 in 2024-09-21T09:56:36.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:36.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:36.463 INFO:tasks.rados.rados.0.smithi033.stdout:1655: done (2 left) 2024-09-21T09:56:36.463 INFO:tasks.rados.rados.0.smithi033.stdout:1656: done (1 left) 2024-09-21T09:56:36.463 INFO:tasks.rados.rados.0.smithi033.stdout:1659: done (0 left) 2024-09-21T09:56:36.463 INFO:tasks.rados.rados.0.smithi033.stdout:1660: rollback oid 7 current snap is 147 2024-09-21T09:56:36.463 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 7 to 146 2024-09-21T09:56:36.476 INFO:tasks.rados.rados.0.smithi033.stdout:1661: snap_remove snap 143 2024-09-21T09:56:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:36 smithi033 ceph-mon[98546]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:56:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:36 smithi033 ceph-mon[98546]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:56:36.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:36 smithi033 ceph-mon[96568]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:56:36.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:36 smithi033 ceph-mon[96568]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:56:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:36 smithi105 ceph-mon[86649]: osdmap e369: 8 total, 8 up, 8 in 2024-09-21T09:56:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:36 smithi105 ceph-mon[86649]: osdmap e370: 8 total, 8 up, 8 in 2024-09-21T09:56:37.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:37.465 INFO:tasks.rados.rados.0.smithi033.stdout:1661: done (1 left) 2024-09-21T09:56:37.465 INFO:tasks.rados.rados.0.smithi033.stdout:1662: rollback oid 16 current snap is 147 2024-09-21T09:56:37.465 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 16 to 146 2024-09-21T09:56:37.465 INFO:tasks.rados.rados.0.smithi033.stdout:1660: finishing rollback tid 0 to smithi03385152-7 2024-09-21T09:56:37.465 INFO:tasks.rados.rados.0.smithi033.stdout:1660: finishing rollback tid 1 to smithi03385152-7 2024-09-21T09:56:37.465 INFO:tasks.rados.rados.0.smithi033.stdout:1660: finishing rollback tid 2 to smithi03385152-7 2024-09-21T09:56:37.465 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 1044 (ObjNum 474 snap 125 seq_num 474) dirty exists 2024-09-21T09:56:37.465 INFO:tasks.rados.rados.0.smithi033.stdout:1660: done (1 left) 2024-09-21T09:56:37.466 INFO:tasks.rados.rados.0.smithi033.stdout:1663: rollback oid 17 current snap is 147 2024-09-21T09:56:37.466 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 146 2024-09-21T09:56:37.466 INFO:tasks.rados.rados.0.smithi033.stdout:1664: read oid 4 snap -1 2024-09-21T09:56:37.466 INFO:tasks.rados.rados.0.smithi033.stdout:1664: expect (ObjNum 502 snap 132 seq_num 502) 2024-09-21T09:56:37.466 INFO:tasks.rados.rados.0.smithi033.stdout:1665: read oid 48 snap -1 2024-09-21T09:56:37.466 INFO:tasks.rados.rados.0.smithi033.stdout:1665: expect (ObjNum 508 snap 135 seq_num 508) 2024-09-21T09:56:37.466 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 147 2024-09-21T09:56:37.466 INFO:tasks.rados.rados.0.smithi033.stdout:1666: seq_num 544 ranges {0=720896,720896=614400,1335296=589824} 2024-09-21T09:56:37.469 INFO:tasks.rados.rados.0.smithi033.stdout:1666: writing smithi03385152-23 from 0 to 720896 tid 1 2024-09-21T09:56:37.473 INFO:tasks.rados.rados.0.smithi033.stdout:1666: writing smithi03385152-23 from 720896 to 1335296 tid 2 2024-09-21T09:56:37.478 INFO:tasks.rados.rados.0.smithi033.stdout:1666: writing smithi03385152-23 from 1335296 to 1925120 tid 3 2024-09-21T09:56:37.478 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 147 2024-09-21T09:56:37.478 INFO:tasks.rados.rados.0.smithi033.stdout:1667: seq_num 545 ranges {2752512=737280,3489792=491520,3981312=450560,4431872=262144} 2024-09-21T09:56:37.498 INFO:tasks.rados.rados.0.smithi033.stdout:1667: writing smithi03385152-35 from 2752512 to 3489792 tid 1 2024-09-21T09:56:37.501 INFO:tasks.rados.rados.0.smithi033.stdout:1667: writing smithi03385152-35 from 3489792 to 3981312 tid 2 2024-09-21T09:56:37.503 INFO:tasks.rados.rados.0.smithi033.stdout:1667: writing smithi03385152-35 from 3981312 to 4431872 tid 3 2024-09-21T09:56:37.505 INFO:tasks.rados.rados.0.smithi033.stdout:1667: writing smithi03385152-35 from 4431872 to 4694016 tid 4 2024-09-21T09:56:37.505 INFO:tasks.rados.rados.0.smithi033.stdout:1668: read oid 12 snap -1 2024-09-21T09:56:37.505 INFO:tasks.rados.rados.0.smithi033.stdout:1668: expect (ObjNum 480 snap 125 seq_num 480) 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:1669: copy_from oid 2 from oid 22 current snap is 147 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:1663: finishing rollback tid 0 to smithi03385152-17 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:1666: finishing write tid 1 to smithi03385152-23 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:1666: finishing write tid 2 to smithi03385152-23 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:1666: finishing write tid 3 to smithi03385152-23 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:1663: finishing rollback tid 1 to smithi03385152-17 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:1663: finishing rollback tid 2 to smithi03385152-17 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 849 (ObjNum 541 snap 145 seq_num 541) dirty exists 2024-09-21T09:56:37.506 INFO:tasks.rados.rados.0.smithi033.stdout:1666: finishing write tid 4 to smithi03385152-23 2024-09-21T09:56:37.507 INFO:tasks.rados.rados.0.smithi033.stdout:1665: expect (ObjNum 508 snap 135 seq_num 508) 2024-09-21T09:56:37.548 INFO:tasks.rados.rados.0.smithi033.stdout:1662: finishing rollback tid 0 to smithi03385152-16 2024-09-21T09:56:37.548 INFO:tasks.rados.rados.0.smithi033.stdout:1662: finishing rollback tid 1 to smithi03385152-16 2024-09-21T09:56:37.548 INFO:tasks.rados.rados.0.smithi033.stdout:1662: finishing rollback tid 2 to smithi03385152-16 2024-09-21T09:56:37.548 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 1052 (ObjNum 507 snap 135 seq_num 507) dirty exists 2024-09-21T09:56:37.548 INFO:tasks.rados.rados.0.smithi033.stdout:1666: finishing write tid 5 to smithi03385152-23 2024-09-21T09:56:37.548 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 707 (ObjNum 544 snap 147 seq_num 544) dirty exists 2024-09-21T09:56:37.549 INFO:tasks.rados.rados.0.smithi033.stdout:1666: left oid 23 (ObjNum 544 snap 147 seq_num 544) 2024-09-21T09:56:37.549 INFO:tasks.rados.rados.0.smithi033.stdout:1664: expect (ObjNum 502 snap 132 seq_num 502) 2024-09-21T09:56:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:37 smithi033 ceph-mon[96568]: pgmap v183: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-21T09:56:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:37 smithi033 ceph-mon[96568]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:56:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:37 smithi033 ceph-mon[98546]: pgmap v183: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-21T09:56:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:37 smithi033 ceph-mon[98546]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:56:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:37 smithi105 ceph-mon[86649]: pgmap v183: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-21T09:56:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:37 smithi105 ceph-mon[86649]: osdmap e371: 8 total, 8 up, 8 in 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:1667: finishing write tid 1 to smithi03385152-35 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:1667: finishing write tid 2 to smithi03385152-35 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:1667: finishing write tid 3 to smithi03385152-35 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:1667: finishing write tid 4 to smithi03385152-35 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:1667: finishing write tid 5 to smithi03385152-35 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:1667: finishing write tid 6 to smithi03385152-35 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 1058 (ObjNum 545 snap 147 seq_num 545) dirty exists 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:1667: left oid 35 (ObjNum 545 snap 147 seq_num 545) 2024-09-21T09:56:37.906 INFO:tasks.rados.rados.0.smithi033.stdout:1668: expect (ObjNum 480 snap 125 seq_num 480) 2024-09-21T09:56:38.017 INFO:tasks.rados.rados.0.smithi033.stdout:1669: finishing copy_from racing read to smithi03385152-2 2024-09-21T09:56:38.017 INFO:tasks.rados.rados.0.smithi033.stdout:1669: finishing copy_from to smithi03385152-2 2024-09-21T09:56:38.017 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 879 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:56:38.017 INFO:tasks.rados.rados.0.smithi033.stdout:1662: done (7 left) 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:1663: done (6 left) 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:1664: done (5 left) 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:1665: done (4 left) 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:1666: done (3 left) 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:1667: done (2 left) 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:1668: done (1 left) 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:1669: done (0 left) 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 147 2024-09-21T09:56:38.018 INFO:tasks.rados.rados.0.smithi033.stdout:1670: seq_num 546 ranges {0=753664,753664=778240,1531904=425984} 2024-09-21T09:56:38.021 INFO:tasks.rados.rados.0.smithi033.stdout:1670: writing smithi03385152-36 from 0 to 753664 tid 1 2024-09-21T09:56:38.025 INFO:tasks.rados.rados.0.smithi033.stdout:1670: writing smithi03385152-36 from 753664 to 1531904 tid 2 2024-09-21T09:56:38.028 INFO:tasks.rados.rados.0.smithi033.stdout:1670: writing smithi03385152-36 from 1531904 to 1957888 tid 3 2024-09-21T09:56:38.028 INFO:tasks.rados.rados.0.smithi033.stdout:1671: snap_remove snap 146 2024-09-21T09:56:38.460 INFO:tasks.rados.rados.0.smithi033.stdout:1671: done (1 left) 2024-09-21T09:56:38.460 INFO:tasks.rados.rados.0.smithi033.stdout:1672: copy_from oid 26 from oid 1 current snap is 147 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:1670: finishing write tid 1 to smithi03385152-36 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:1670: finishing write tid 2 to smithi03385152-36 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:1670: finishing write tid 3 to smithi03385152-36 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:1670: finishing write tid 4 to smithi03385152-36 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:1670: finishing write tid 5 to smithi03385152-36 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1062 (ObjNum 546 snap 147 seq_num 546) dirty exists 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:1670: left oid 36 (ObjNum 546 snap 147 seq_num 546) 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:1670: done (1 left) 2024-09-21T09:56:38.461 INFO:tasks.rados.rados.0.smithi033.stdout:1673: snap_create 2024-09-21T09:56:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1672: finishing copy_from racing read to smithi03385152-26 2024-09-21T09:56:38.475 INFO:tasks.rados.rados.0.smithi033.stdout:1672: finishing copy_from to smithi03385152-26 2024-09-21T09:56:38.475 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 652 (ObjNum 404 snap 110 seq_num 404) dirty exists 2024-09-21T09:56:39.466 INFO:tasks.rados.rados.0.smithi033.stdout:1672: done (1 left) 2024-09-21T09:56:39.466 INFO:tasks.rados.rados.0.smithi033.stdout:1673: done (0 left) 2024-09-21T09:56:39.466 INFO:tasks.rados.rados.0.smithi033.stdout:1674: read oid 44 snap -1 2024-09-21T09:56:39.466 INFO:tasks.rados.rados.0.smithi033.stdout:1674: expect (ObjNum 524 snap 142 seq_num 524) 2024-09-21T09:56:39.466 INFO:tasks.rados.rados.0.smithi033.stdout:1675: read oid 11 snap -1 2024-09-21T09:56:39.466 INFO:tasks.rados.rados.0.smithi033.stdout:1675: expect deleted 2024-09-21T09:56:39.466 INFO:tasks.rados.rados.0.smithi033.stdout:1676: copy_from oid 33 from oid 6 current snap is 148 2024-09-21T09:56:39.466 INFO:tasks.rados.rados.0.smithi033.stdout:1677: snap_remove snap 147 2024-09-21T09:56:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:39 smithi033 ceph-mon[98546]: pgmap v186: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-21T09:56:39.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:39 smithi033 ceph-mon[98546]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T09:56:39.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:39 smithi033 ceph-mon[96568]: pgmap v186: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-21T09:56:39.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:39 smithi033 ceph-mon[96568]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T09:56:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:39 smithi105 ceph-mon[86649]: pgmap v186: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 17 op/s 2024-09-21T09:56:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:39 smithi105 ceph-mon[86649]: osdmap e372: 8 total, 8 up, 8 in 2024-09-21T09:56:40.468 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:40.478 INFO:tasks.rados.rados.0.smithi033.stdout:1676: finishing copy_from racing read to smithi03385152-33 2024-09-21T09:56:40.478 INFO:tasks.rados.rados.0.smithi033.stdout:1676: finishing copy_from to smithi03385152-33 2024-09-21T09:56:40.478 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1016 (ObjNum 527 snap 143 seq_num 527) dirty exists 2024-09-21T09:56:40.478 INFO:tasks.rados.rados.0.smithi033.stdout:1674: expect (ObjNum 524 snap 142 seq_num 524) 2024-09-21T09:56:40.623 INFO:tasks.rados.rados.0.smithi033.stdout:1674: done (3 left) 2024-09-21T09:56:40.624 INFO:tasks.rados.rados.0.smithi033.stdout:1675: done (2 left) 2024-09-21T09:56:40.624 INFO:tasks.rados.rados.0.smithi033.stdout:1676: done (1 left) 2024-09-21T09:56:40.624 INFO:tasks.rados.rados.0.smithi033.stdout:1677: done (0 left) 2024-09-21T09:56:40.624 INFO:tasks.rados.rados.0.smithi033.stdout:1678: read oid 6 snap -1 2024-09-21T09:56:40.624 INFO:tasks.rados.rados.0.smithi033.stdout:1678: expect (ObjNum 527 snap 143 seq_num 527) 2024-09-21T09:56:40.624 INFO:tasks.rados.rados.0.smithi033.stdout:1679: delete oid 24 current snap is 148 2024-09-21T09:56:40.626 INFO:tasks.rados.rados.0.smithi033.stdout:1679: done (1 left) 2024-09-21T09:56:40.626 INFO:tasks.rados.rados.0.smithi033.stdout:1680: read oid 28 snap -1 2024-09-21T09:56:40.626 INFO:tasks.rados.rados.0.smithi033.stdout:1680: expect deleted 2024-09-21T09:56:40.626 INFO:tasks.rados.rados.0.smithi033.stdout:1681: read oid 7 snap -1 2024-09-21T09:56:40.626 INFO:tasks.rados.rados.0.smithi033.stdout:1681: expect (ObjNum 474 snap 125 seq_num 474) 2024-09-21T09:56:40.626 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 148 2024-09-21T09:56:40.626 INFO:tasks.rados.rados.0.smithi033.stdout:1682: seq_num 548 ranges {2899968=786432,3686400=598016,4284416=606208} 2024-09-21T09:56:40.646 INFO:tasks.rados.rados.0.smithi033.stdout:1682: writing smithi03385152-31 from 2899968 to 3686400 tid 1 2024-09-21T09:56:40.649 INFO:tasks.rados.rados.0.smithi033.stdout:1682: writing smithi03385152-31 from 3686400 to 4284416 tid 2 2024-09-21T09:56:40.654 INFO:tasks.rados.rados.0.smithi033.stdout:1682: writing smithi03385152-31 from 4284416 to 4890624 tid 3 2024-09-21T09:56:40.654 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 148 2024-09-21T09:56:40.654 INFO:tasks.rados.rados.0.smithi033.stdout:1683: seq_num 549 ranges {6758400=802816,7561216=483328,8044544=434176,8478720=286720} 2024-09-21T09:56:40.695 INFO:tasks.rados.rados.0.smithi033.stdout:1683: writing smithi03385152-15 from 6758400 to 7561216 tid 1 2024-09-21T09:56:40.697 INFO:tasks.rados.rados.0.smithi033.stdout:1683: writing smithi03385152-15 from 7561216 to 8044544 tid 2 2024-09-21T09:56:40.700 INFO:tasks.rados.rados.0.smithi033.stdout:1683: writing smithi03385152-15 from 8044544 to 8478720 tid 3 2024-09-21T09:56:40.702 INFO:tasks.rados.rados.0.smithi033.stdout:1683: writing smithi03385152-15 from 8478720 to 8765440 tid 4 2024-09-21T09:56:40.702 INFO:tasks.rados.rados.0.smithi033.stdout:1684: snap_create 2024-09-21T09:56:40.702 INFO:tasks.rados.rados.0.smithi033.stdout:1678: expect (ObjNum 527 snap 143 seq_num 527) 2024-09-21T09:56:40.745 INFO:tasks.rados.rados.0.smithi033.stdout:1681: expect (ObjNum 474 snap 125 seq_num 474) 2024-09-21T09:56:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:40 smithi033 ceph-mon[98546]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:56:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:40 smithi033 ceph-mon[96568]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1682: finishing write tid 1 to smithi03385152-31 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1682: finishing write tid 2 to smithi03385152-31 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1682: finishing write tid 3 to smithi03385152-31 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1682: finishing write tid 4 to smithi03385152-31 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1682: finishing write tid 5 to smithi03385152-31 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1072 (ObjNum 548 snap 148 seq_num 548) dirty exists 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1682: left oid 31 (ObjNum 548 snap 148 seq_num 548) 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1683: finishing write tid 1 to smithi03385152-15 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1683: finishing write tid 2 to smithi03385152-15 2024-09-21T09:56:40.765 INFO:tasks.rados.rados.0.smithi033.stdout:1683: finishing write tid 3 to smithi03385152-15 2024-09-21T09:56:40.766 INFO:tasks.rados.rados.0.smithi033.stdout:1683: finishing write tid 4 to smithi03385152-15 2024-09-21T09:56:40.766 INFO:tasks.rados.rados.0.smithi033.stdout:1683: finishing write tid 5 to smithi03385152-15 2024-09-21T09:56:40.766 INFO:tasks.rados.rados.0.smithi033.stdout:1683: finishing write tid 6 to smithi03385152-15 2024-09-21T09:56:40.766 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 670 (ObjNum 549 snap 148 seq_num 549) dirty exists 2024-09-21T09:56:40.766 INFO:tasks.rados.rados.0.smithi033.stdout:1683: left oid 15 (ObjNum 549 snap 148 seq_num 549) 2024-09-21T09:56:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:40 smithi105 ceph-mon[86649]: osdmap e373: 8 total, 8 up, 8 in 2024-09-21T09:56:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1678: done (5 left) 2024-09-21T09:56:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1680: done (4 left) 2024-09-21T09:56:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1681: done (3 left) 2024-09-21T09:56:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1682: done (2 left) 2024-09-21T09:56:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:1683: done (1 left) 2024-09-21T09:56:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:1684: done (0 left) 2024-09-21T09:56:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:1685: rmattr oid 13 current snap is 149 2024-09-21T09:56:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:1686: read oid 26 snap 148 2024-09-21T09:56:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:1686: expect (ObjNum 404 snap 110 seq_num 404) 2024-09-21T09:56:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 149 2024-09-21T09:56:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:1687: seq_num 550 ranges {1638400=417792,2056192=770048,2826240=638976,3465216=196608} 2024-09-21T09:56:41.487 INFO:tasks.rados.rados.0.smithi033.stdout:1687: writing smithi03385152-6 from 1638400 to 2056192 tid 1 2024-09-21T09:56:41.491 INFO:tasks.rados.rados.0.smithi033.stdout:1687: writing smithi03385152-6 from 2056192 to 2826240 tid 2 2024-09-21T09:56:41.495 INFO:tasks.rados.rados.0.smithi033.stdout:1687: writing smithi03385152-6 from 2826240 to 3465216 tid 3 2024-09-21T09:56:41.496 INFO:tasks.rados.rados.0.smithi033.stdout:1687: writing smithi03385152-6 from 3465216 to 3661824 tid 4 2024-09-21T09:56:41.496 INFO:tasks.rados.rados.0.smithi033.stdout:1688: rollback oid 24 current snap is 149 2024-09-21T09:56:41.496 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 148 2024-09-21T09:56:41.496 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 1074 (ObjNum 414 snap 110 seq_num 414) dirty exists 2024-09-21T09:56:41.496 INFO:tasks.rados.rados.0.smithi033.stdout:1686: expect (ObjNum 404 snap 110 seq_num 404) 2024-09-21T09:56:41.549 INFO:tasks.rados.rados.0.smithi033.stdout:1687: finishing write tid 1 to smithi03385152-6 2024-09-21T09:56:41.549 INFO:tasks.rados.rados.0.smithi033.stdout:1687: finishing write tid 2 to smithi03385152-6 2024-09-21T09:56:41.549 INFO:tasks.rados.rados.0.smithi033.stdout:1688: finishing rollback tid 1 to smithi03385152-24 2024-09-21T09:56:41.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 0 (ObjNum 177 snap 0 seq_num 90597104) dirty dne 2024-09-21T09:56:41.549 INFO:tasks.rados.rados.0.smithi033.stdout:1687: finishing write tid 3 to smithi03385152-6 2024-09-21T09:56:41.549 INFO:tasks.rados.rados.0.smithi033.stdout:1687: finishing write tid 4 to smithi03385152-6 2024-09-21T09:56:41.549 INFO:tasks.rados.rados.0.smithi033.stdout:1685: done (3 left) 2024-09-21T09:56:41.549 INFO:tasks.rados.rados.0.smithi033.stdout:1686: done (2 left) 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1688: done (1 left) 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1689: read oid 20 snap -1 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1689: expect (ObjNum 532 snap 143 seq_num 532) 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1690: read oid 3 snap -1 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1690: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 149 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1687: finishing write tid 5 to smithi03385152-6 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1687: finishing write tid 6 to smithi03385152-6 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1021 (ObjNum 550 snap 149 seq_num 550) dirty exists 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1687: left oid 6 (ObjNum 550 snap 149 seq_num 550) 2024-09-21T09:56:41.550 INFO:tasks.rados.rados.0.smithi033.stdout:1691: seq_num 551 ranges {0=434176,434176=647168,1081344=450560,1531904=507904} 2024-09-21T09:56:41.551 INFO:tasks.rados.rados.0.smithi033.stdout:1691: writing smithi03385152-14 from 0 to 434176 tid 1 2024-09-21T09:56:41.554 INFO:tasks.rados.rados.0.smithi033.stdout:1691: writing smithi03385152-14 from 434176 to 1081344 tid 2 2024-09-21T09:56:41.557 INFO:tasks.rados.rados.0.smithi033.stdout:1691: writing smithi03385152-14 from 1081344 to 1531904 tid 3 2024-09-21T09:56:41.560 INFO:tasks.rados.rados.0.smithi033.stdout:1691: writing smithi03385152-14 from 1531904 to 2039808 tid 4 2024-09-21T09:56:41.560 INFO:tasks.rados.rados.0.smithi033.stdout:1687: done (3 left) 2024-09-21T09:56:41.560 INFO:tasks.rados.rados.0.smithi033.stdout:1692: copy_from oid 10 from oid 44 current snap is 149 2024-09-21T09:56:41.560 INFO:tasks.rados.rados.0.smithi033.stdout:1690: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:56:41.569 INFO:tasks.rados.rados.0.smithi033.stdout:1691: finishing write tid 1 to smithi03385152-14 2024-09-21T09:56:41.569 INFO:tasks.rados.rados.0.smithi033.stdout:1691: finishing write tid 2 to smithi03385152-14 2024-09-21T09:56:41.570 INFO:tasks.rados.rados.0.smithi033.stdout:1689: expect (ObjNum 532 snap 143 seq_num 532) 2024-09-21T09:56:41.664 INFO:tasks.rados.rados.0.smithi033.stdout:1691: finishing write tid 3 to smithi03385152-14 2024-09-21T09:56:41.664 INFO:tasks.rados.rados.0.smithi033.stdout:1691: finishing write tid 4 to smithi03385152-14 2024-09-21T09:56:41.664 INFO:tasks.rados.rados.0.smithi033.stdout:1691: finishing write tid 5 to smithi03385152-14 2024-09-21T09:56:41.664 INFO:tasks.rados.rados.0.smithi033.stdout:1691: finishing write tid 6 to smithi03385152-14 2024-09-21T09:56:41.664 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 905 (ObjNum 551 snap 149 seq_num 551) dirty exists 2024-09-21T09:56:41.664 INFO:tasks.rados.rados.0.smithi033.stdout:1691: left oid 14 (ObjNum 551 snap 149 seq_num 551) 2024-09-21T09:56:41.664 INFO:tasks.rados.rados.0.smithi033.stdout:1692: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1692: finishing copy_from to smithi03385152-10 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 998 (ObjNum 524 snap 142 seq_num 524) dirty exists 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1689: done (3 left) 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1690: done (2 left) 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1691: done (1 left) 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1692: done (0 left) 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1693: rollback oid 25 current snap is 149 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 25 to 148 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1694: rollback oid 18 current snap is 149 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 148 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1695: read oid 40 snap -1 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1695: expect (ObjNum 459 snap 121 seq_num 459) 2024-09-21T09:56:41.665 INFO:tasks.rados.rados.0.smithi033.stdout:1696: delete oid 43 current snap is 149 2024-09-21T09:56:41.668 INFO:tasks.rados.rados.0.smithi033.stdout:1694: finishing rollback tid 0 to smithi03385152-18 2024-09-21T09:56:41.669 INFO:tasks.rados.rados.0.smithi033.stdout:1694: finishing rollback tid 1 to smithi03385152-18 2024-09-21T09:56:41.669 INFO:tasks.rados.rados.0.smithi033.stdout:1696: done (3 left) 2024-09-21T09:56:41.669 INFO:tasks.rados.rados.0.smithi033.stdout:1697: rollback oid 21 current snap is 149 2024-09-21T09:56:41.669 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 148 2024-09-21T09:56:41.669 INFO:tasks.rados.rados.0.smithi033.stdout:1698: rollback oid 34 current snap is 149 2024-09-21T09:56:41.670 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 148 2024-09-21T09:56:41.670 INFO:tasks.rados.rados.0.smithi033.stdout:1699: rmattr oid 30 current snap is 149 2024-09-21T09:56:41.670 INFO:tasks.rados.rados.0.smithi033.stdout:1699: done (5 left) 2024-09-21T09:56:41.670 INFO:tasks.rados.rados.0.smithi033.stdout:1700: rollback oid 36 current snap is 149 2024-09-21T09:56:41.670 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 148 2024-09-21T09:56:41.670 INFO:tasks.rados.rados.0.smithi033.stdout:1701: setattr oid 22 current snap is 149 2024-09-21T09:56:41.671 INFO:tasks.rados.rados.0.smithi033.stdout:1702: read oid 16 snap -1 2024-09-21T09:56:41.671 INFO:tasks.rados.rados.0.smithi033.stdout:1702: expect (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:41.671 INFO:tasks.rados.rados.0.smithi033.stdout:1703: rollback oid 31 current snap is 149 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 31 to 148 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1704: rmattr oid 30 current snap is 149 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1704: done (9 left) 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1705: delete oid 9 current snap is 149 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1693: finishing rollback tid 0 to smithi03385152-25 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1697: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 0 (ObjNum 33 snap 0 seq_num 2) dirty dne 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1694: finishing rollback tid 2 to smithi03385152-18 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 136 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1693: finishing rollback tid 1 to smithi03385152-25 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1693: finishing rollback tid 2 to smithi03385152-25 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 882 (ObjNum 535 snap 143 seq_num 535) dirty exists 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:1698: finishing rollback tid 1 to smithi03385152-34 2024-09-21T09:56:41.672 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 0 (ObjNum 177 snap 0 seq_num 0) dirty dne 2024-09-21T09:56:41.673 INFO:tasks.rados.rados.0.smithi033.stdout:1700: finishing rollback tid 0 to smithi03385152-36 2024-09-21T09:56:41.675 INFO:tasks.rados.rados.0.smithi033.stdout:1695: expect (ObjNum 459 snap 121 seq_num 459) 2024-09-21T09:56:41.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:56:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:56:41] "GET /metrics HTTP/1.1" 200 36246 "" "Prometheus/2.51.0" 2024-09-21T09:56:41.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:41 smithi033 ceph-mon[98546]: pgmap v189: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:41.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:41 smithi033 ceph-mon[98546]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:56:41.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:41 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:41.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:41 smithi033 ceph-mon[96568]: pgmap v189: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:41.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:41 smithi033 ceph-mon[96568]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:56:41.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:41 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:41.776 INFO:tasks.rados.rados.0.smithi033.stdout:1700: finishing rollback tid 1 to smithi03385152-36 2024-09-21T09:56:41.777 INFO:tasks.rados.rados.0.smithi033.stdout:1700: finishing rollback tid 2 to smithi03385152-36 2024-09-21T09:56:41.777 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1078 (ObjNum 546 snap 147 seq_num 546) dirty exists 2024-09-21T09:56:41.777 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 908 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:56:41.777 INFO:tasks.rados.rados.0.smithi033.stdout:1703: finishing rollback tid 0 to smithi03385152-31 2024-09-21T09:56:41.777 INFO:tasks.rados.rados.0.smithi033.stdout:1702: expect (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:41 smithi105 ceph-mon[86649]: pgmap v189: 121 pgs: 121 active+clean; 132 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-21T09:56:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:41 smithi105 ceph-mon[86649]: osdmap e374: 8 total, 8 up, 8 in 2024-09-21T09:56:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:41 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:42.348 INFO:tasks.rados.rados.0.smithi033.stdout:1703: finishing rollback tid 1 to smithi03385152-31 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1703: finishing rollback tid 2 to smithi03385152-31 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1082 (ObjNum 548 snap 148 seq_num 548) dirty exists 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1693: done (9 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1694: done (8 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1695: done (7 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1697: done (6 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1698: done (5 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1700: done (4 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1701: done (3 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1702: done (2 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1703: done (1 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:1705: done (0 left) 2024-09-21T09:56:42.349 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 149 2024-09-21T09:56:42.350 INFO:tasks.rados.rados.0.smithi033.stdout:1706: seq_num 554 ranges {1974272=483328,2457600=704512,3162112=622592,3784704=278528} 2024-09-21T09:56:42.361 INFO:tasks.rados.rados.0.smithi033.stdout:1706: writing smithi03385152-26 from 1974272 to 2457600 tid 1 2024-09-21T09:56:42.365 INFO:tasks.rados.rados.0.smithi033.stdout:1706: writing smithi03385152-26 from 2457600 to 3162112 tid 2 2024-09-21T09:56:42.368 INFO:tasks.rados.rados.0.smithi033.stdout:1706: writing smithi03385152-26 from 3162112 to 3784704 tid 3 2024-09-21T09:56:42.370 INFO:tasks.rados.rados.0.smithi033.stdout:1706: writing smithi03385152-26 from 3784704 to 4063232 tid 4 2024-09-21T09:56:42.370 INFO:tasks.rados.rados.0.smithi033.stdout:1707: copy_from oid 1 from oid 28 current snap is 149 2024-09-21T09:56:42.370 INFO:tasks.rados.rados.0.smithi033.stdout:1706: finishing write tid 1 to smithi03385152-26 2024-09-21T09:56:42.371 INFO:tasks.rados.rados.0.smithi033.stdout:1706: finishing write tid 2 to smithi03385152-26 2024-09-21T09:56:42.371 INFO:tasks.rados.rados.0.smithi033.stdout:1708: snap_create 2024-09-21T09:56:42.372 INFO:tasks.rados.rados.0.smithi033.stdout:1707: finishing copy_from to smithi03385152-1 2024-09-21T09:56:42.372 INFO:tasks.rados.rados.0.smithi033.stdout:1707: got expected ENOENT (src dne) 2024-09-21T09:56:42.372 INFO:tasks.rados.rados.0.smithi033.stdout:1707: finishing copy_from racing read to smithi03385152-1 2024-09-21T09:56:42.373 INFO:tasks.rados.rados.0.smithi033.stdout:1706: finishing write tid 3 to smithi03385152-26 2024-09-21T09:56:42.373 INFO:tasks.rados.rados.0.smithi033.stdout:1706: finishing write tid 4 to smithi03385152-26 2024-09-21T09:56:42.373 INFO:tasks.rados.rados.0.smithi033.stdout:1706: finishing write tid 5 to smithi03385152-26 2024-09-21T09:56:42.374 INFO:tasks.rados.rados.0.smithi033.stdout:1706: finishing write tid 6 to smithi03385152-26 2024-09-21T09:56:42.374 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 854 (ObjNum 554 snap 149 seq_num 554) dirty exists 2024-09-21T09:56:42.374 INFO:tasks.rados.rados.0.smithi033.stdout:1706: left oid 26 (ObjNum 554 snap 149 seq_num 554) 2024-09-21T09:56:42.493 INFO:tasks.rados.rados.0.smithi033.stdout:1706: done (2 left) 2024-09-21T09:56:42.493 INFO:tasks.rados.rados.0.smithi033.stdout:1707: done (1 left) 2024-09-21T09:56:42.493 INFO:tasks.rados.rados.0.smithi033.stdout:1708: done (0 left) 2024-09-21T09:56:42.493 INFO:tasks.rados.rados.0.smithi033.stdout:1709: rollback oid 2 current snap is 150 2024-09-21T09:56:42.493 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 149 2024-09-21T09:56:42.493 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 150 2024-09-21T09:56:42.493 INFO:tasks.rados.rados.0.smithi033.stdout:1710: seq_num 555 ranges {0=499712,499712=638976,1138688=425984,1564672=540672} 2024-09-21T09:56:42.496 INFO:tasks.rados.rados.0.smithi033.stdout:1710: writing smithi03385152-11 from 0 to 499712 tid 1 2024-09-21T09:56:42.500 INFO:tasks.rados.rados.0.smithi033.stdout:1710: writing smithi03385152-11 from 499712 to 1138688 tid 2 2024-09-21T09:56:42.503 INFO:tasks.rados.rados.0.smithi033.stdout:1710: writing smithi03385152-11 from 1138688 to 1564672 tid 3 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1710: writing smithi03385152-11 from 1564672 to 2105344 tid 4 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1711: copy_from oid 49 from oid 36 current snap is 150 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1709: finishing rollback tid 0 to smithi03385152-2 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1709: finishing rollback tid 1 to smithi03385152-2 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1709: finishing rollback tid 2 to smithi03385152-2 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 885 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1710: finishing write tid 1 to smithi03385152-11 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1710: finishing write tid 2 to smithi03385152-11 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1710: finishing write tid 3 to smithi03385152-11 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1709: done (2 left) 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 150 2024-09-21T09:56:42.507 INFO:tasks.rados.rados.0.smithi033.stdout:1712: seq_num 556 ranges {0=516096,516096=524288,1040384=638976,1679360=442368} 2024-09-21T09:56:42.509 INFO:tasks.rados.rados.0.smithi033.stdout:1712: writing smithi03385152-43 from 0 to 516096 tid 1 2024-09-21T09:56:42.512 INFO:tasks.rados.rados.0.smithi033.stdout:1712: writing smithi03385152-43 from 516096 to 1040384 tid 2 2024-09-21T09:56:42.516 INFO:tasks.rados.rados.0.smithi033.stdout:1712: writing smithi03385152-43 from 1040384 to 1679360 tid 3 2024-09-21T09:56:42.520 INFO:tasks.rados.rados.0.smithi033.stdout:1712: writing smithi03385152-43 from 1679360 to 2121728 tid 4 2024-09-21T09:56:42.520 INFO:tasks.rados.rados.0.smithi033.stdout:1713: read oid 37 snap -1 2024-09-21T09:56:42.520 INFO:tasks.rados.rados.0.smithi033.stdout:1713: expect (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:56:42.520 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 150 2024-09-21T09:56:42.520 INFO:tasks.rados.rados.0.smithi033.stdout:1714: seq_num 557 ranges {1638400=532480,2170880=802816,2973696=794624,3768320=8192} 2024-09-21T09:56:42.533 INFO:tasks.rados.rados.0.smithi033.stdout:1714: writing smithi03385152-33 from 1638400 to 2170880 tid 1 2024-09-21T09:56:42.538 INFO:tasks.rados.rados.0.smithi033.stdout:1714: writing smithi03385152-33 from 2170880 to 2973696 tid 2 2024-09-21T09:56:42.543 INFO:tasks.rados.rados.0.smithi033.stdout:1714: writing smithi03385152-33 from 2973696 to 3768320 tid 3 2024-09-21T09:56:42.543 INFO:tasks.rados.rados.0.smithi033.stdout:1714: writing smithi03385152-33 from 3768320 to 3776512 tid 4 2024-09-21T09:56:42.543 INFO:tasks.rados.rados.0.smithi033.stdout:1715: snap_remove snap 148 2024-09-21T09:56:42.565 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:42 smithi033 ceph-mon[98546]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:56:42.565 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:42 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:56:42.565 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:42 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:56:42.565 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:42 smithi033 ceph-mon[98546]: Upgrade: osd.2 is safe to restart 2024-09-21T09:56:42.566 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:42 smithi033 ceph-mon[96568]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:56:42.566 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:42 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:56:42.566 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:42 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:56:42.566 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:42 smithi033 ceph-mon[96568]: Upgrade: osd.2 is safe to restart 2024-09-21T09:56:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:42 smithi105 ceph-mon[86649]: osdmap e375: 8 total, 8 up, 8 in 2024-09-21T09:56:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:42 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-21T09:56:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:42 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-21T09:56:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:42 smithi105 ceph-mon[86649]: Upgrade: osd.2 is safe to restart 2024-09-21T09:56:43.460 INFO:tasks.rados.rados.0.smithi033.stdout:1715: done (5 left) 2024-09-21T09:56:43.460 INFO:tasks.rados.rados.0.smithi033.stdout:1716: rollback oid 42 current snap is 150 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 42 to 149 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1717: delete oid 45 current snap is 150 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1710: finishing write tid 4 to smithi03385152-11 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1710: finishing write tid 5 to smithi03385152-11 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1710: finishing write tid 6 to smithi03385152-11 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 717 (ObjNum 555 snap 150 seq_num 555) dirty exists 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1710: left oid 11 (ObjNum 555 snap 150 seq_num 555) 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1712: finishing write tid 1 to smithi03385152-43 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1712: finishing write tid 2 to smithi03385152-43 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1711: finishing copy_from racing read to smithi03385152-49 2024-09-21T09:56:43.461 INFO:tasks.rados.rados.0.smithi033.stdout:1712: finishing write tid 3 to smithi03385152-43 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1711: finishing copy_from to smithi03385152-49 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1078 (ObjNum 546 snap 147 seq_num 546) dirty exists 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1712: finishing write tid 4 to smithi03385152-43 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1712: finishing write tid 5 to smithi03385152-43 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1712: finishing write tid 6 to smithi03385152-43 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 722 (ObjNum 556 snap 150 seq_num 556) dirty exists 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1712: left oid 43 (ObjNum 556 snap 150 seq_num 556) 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1714: finishing write tid 1 to smithi03385152-33 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1714: finishing write tid 2 to smithi03385152-33 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1714: finishing write tid 3 to smithi03385152-33 2024-09-21T09:56:43.462 INFO:tasks.rados.rados.0.smithi033.stdout:1713: expect (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:56:43.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[98546]: pgmap v192: 121 pgs: 121 active+clean; 146 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.4 MiB/s wr, 15 op/s 2024-09-21T09:56:43.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[98546]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:56:43.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:43.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:56:43.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:43.714 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[98546]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:56:43.715 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[96568]: pgmap v192: 121 pgs: 121 active+clean; 146 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.4 MiB/s wr, 15 op/s 2024-09-21T09:56:43.715 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[96568]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:56:43.715 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:43.715 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:56:43.715 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:43.715 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:43 smithi033 ceph-mon[96568]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:56:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:43 smithi105 ceph-mon[86649]: pgmap v192: 121 pgs: 121 active+clean; 146 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.4 MiB/s wr, 15 op/s 2024-09-21T09:56:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:43 smithi105 ceph-mon[86649]: osdmap e376: 8 total, 8 up, 8 in 2024-09-21T09:56:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-21T09:56:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:56:43.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:43 smithi105 ceph-mon[86649]: osdmap e377: 8 total, 8 up, 8 in 2024-09-21T09:56:43.868 INFO:tasks.rados.rados.0.smithi033.stdout:1714: finishing write tid 4 to smithi03385152-33 2024-09-21T09:56:43.868 INFO:tasks.rados.rados.0.smithi033.stdout:1714: finishing write tid 5 to smithi03385152-33 2024-09-21T09:56:43.868 INFO:tasks.rados.rados.0.smithi033.stdout:1714: finishing write tid 6 to smithi03385152-33 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1026 (ObjNum 557 snap 150 seq_num 557) dirty exists 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1714: left oid 33 (ObjNum 557 snap 150 seq_num 557) 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1710: done (6 left) 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1711: done (5 left) 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1712: done (4 left) 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1713: done (3 left) 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1714: done (2 left) 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1717: done (1 left) 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1718: delete oid 10 current snap is 150 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1716: finishing rollback tid 0 to smithi03385152-42 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1716: finishing rollback tid 1 to smithi03385152-42 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:1716: finishing rollback tid 2 to smithi03385152-42 2024-09-21T09:56:43.869 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 736 (ObjNum 543 snap 145 seq_num 543) dirty exists 2024-09-21T09:56:43.871 INFO:tasks.rados.rados.0.smithi033.stdout:1716: done (1 left) 2024-09-21T09:56:43.871 INFO:tasks.rados.rados.0.smithi033.stdout:1718: done (0 left) 2024-09-21T09:56:43.871 INFO:tasks.rados.rados.0.smithi033.stdout:1719: snap_remove snap 149 2024-09-21T09:56:44.324 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:56:44.492 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:44.497 INFO:tasks.rados.rados.0.smithi033.stdout:1719: done (0 left) 2024-09-21T09:56:44.497 INFO:tasks.rados.rados.0.smithi033.stdout:1720: setattr oid 36 current snap is 150 2024-09-21T09:56:44.499 INFO:tasks.rados.rados.0.smithi033.stdout:1721: copy_from oid 38 from oid 1 current snap is 150 2024-09-21T09:56:44.499 INFO:tasks.rados.rados.0.smithi033.stdout:1722: rollback oid 15 current snap is 150 2024-09-21T09:56:44.499 INFO:tasks.rados.rados.0.smithi033.stdout:1722: done (2 left) 2024-09-21T09:56:44.499 INFO:tasks.rados.rados.0.smithi033.stdout:1723: read oid 11 snap -1 2024-09-21T09:56:44.499 INFO:tasks.rados.rados.0.smithi033.stdout:1723: expect (ObjNum 555 snap 150 seq_num 555) 2024-09-21T09:56:44.499 INFO:tasks.rados.rados.0.smithi033.stdout:1724: rollback oid 23 current snap is 150 2024-09-21T09:56:44.499 INFO:tasks.rados.rados.0.smithi033.stdout:1724: done (3 left) 2024-09-21T09:56:44.499 INFO:tasks.rados.rados.0.smithi033.stdout:1725: snap_create 2024-09-21T09:56:44.507 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1089 (ObjNum 546 snap 147 seq_num 546) dirty exists 2024-09-21T09:56:44.512 INFO:tasks.rados.rados.0.smithi033.stdout:1721: finishing copy_from racing read to smithi03385152-38 2024-09-21T09:56:44.515 INFO:tasks.rados.rados.0.smithi033.stdout:1721: finishing copy_from to smithi03385152-38 2024-09-21T09:56:44.516 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 652 (ObjNum 404 snap 110 seq_num 404) dirty exists 2024-09-21T09:56:44.518 INFO:tasks.rados.rados.0.smithi033.stdout:1723: expect (ObjNum 555 snap 150 seq_num 555) 2024-09-21T09:56:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:44 smithi033 ceph-mon[98546]: Upgrade: Updating osd.2 2024-09-21T09:56:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:44 smithi033 ceph-mon[98546]: Deploying daemon osd.2 on smithi033 2024-09-21T09:56:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:44 smithi033 ceph-mon[96568]: Upgrade: Updating osd.2 2024-09-21T09:56:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:44 smithi033 ceph-mon[96568]: Deploying daemon osd.2 on smithi033 2024-09-21T09:56:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:44 smithi105 ceph-mon[86649]: Upgrade: Updating osd.2 2024-09-21T09:56:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:44 smithi105 ceph-mon[86649]: Deploying daemon osd.2 on smithi033 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (3m) 22s ago 8m 20.1M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (3m) 97s ago 7m 82.4M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (9m) 22s ago 9m 15.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (9m) 97s ago 9m 19.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (2m) 97s ago 12m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (3m) 22s ago 14m 519M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (2m) 22s ago 14m 49.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (102s) 97s ago 13m 22.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (2m) 22s ago 13m 42.2M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:56:44.967 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (3m) 22s ago 7m 11.6M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (3m) 97s ago 7m 9915k - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (60s) 22s ago 12m 105M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (26s) 22s ago 12m 14.4M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (11m) 22s ago 11m 179M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 56b187b7ae4c 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (11m) 22s ago 11m 106M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (11m) 97s ago 11m 167M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (10m) 97s ago 10m 193M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (10m) 97s ago 10m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (9m) 97s ago 9m 140M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:56:44.968 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (2m) 97s ago 8m 44.0M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:56:45.349 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:56:45.349 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 6, 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:56:45.350 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:56:45.351 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 8, 2024-09-21T09:56:45.351 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 7 2024-09-21T09:56:45.351 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:56:45.351 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:56:45.497 INFO:tasks.rados.rados.0.smithi033.stdout:1720: done (3 left) 2024-09-21T09:56:45.497 INFO:tasks.rados.rados.0.smithi033.stdout:1721: done (2 left) 2024-09-21T09:56:45.497 INFO:tasks.rados.rados.0.smithi033.stdout:1723: done (1 left) 2024-09-21T09:56:45.497 INFO:tasks.rados.rados.0.smithi033.stdout:1725: done (0 left) 2024-09-21T09:56:45.497 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 151 2024-09-21T09:56:45.497 INFO:tasks.rados.rados.0.smithi033.stdout:1726: seq_num 560 ranges {6471680=581632,7053312=458752,7512064=614400,8126464=532480} 2024-09-21T09:56:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:1726: writing smithi03385152-8 from 6471680 to 7053312 tid 1 2024-09-21T09:56:45.537 INFO:tasks.rados.rados.0.smithi033.stdout:1726: writing smithi03385152-8 from 7053312 to 7512064 tid 2 2024-09-21T09:56:45.541 INFO:tasks.rados.rados.0.smithi033.stdout:1726: writing smithi03385152-8 from 7512064 to 8126464 tid 3 2024-09-21T09:56:45.545 INFO:tasks.rados.rados.0.smithi033.stdout:1726: writing smithi03385152-8 from 8126464 to 8658944 tid 4 2024-09-21T09:56:45.545 INFO:tasks.rados.rados.0.smithi033.stdout:1727: delete oid 12 current snap is 151 2024-09-21T09:56:45.545 INFO:tasks.rados.rados.0.smithi033.stdout:1726: finishing write tid 1 to smithi03385152-8 2024-09-21T09:56:45.545 INFO:tasks.rados.rados.0.smithi033.stdout:1726: finishing write tid 2 to smithi03385152-8 2024-09-21T09:56:45.545 INFO:tasks.rados.rados.0.smithi033.stdout:1726: finishing write tid 3 to smithi03385152-8 2024-09-21T09:56:45.547 INFO:tasks.rados.rados.0.smithi033.stdout:1727: done (1 left) 2024-09-21T09:56:45.548 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 151 2024-09-21T09:56:45.548 INFO:tasks.rados.rados.0.smithi033.stdout:1728: seq_num 562 ranges {4694016=622592,5316608=630784,5947392=581632,6529024=393216} 2024-09-21T09:56:45.577 INFO:tasks.rados.rados.0.smithi033.stdout:1728: writing smithi03385152-35 from 4694016 to 5316608 tid 1 2024-09-21T09:56:45.580 INFO:tasks.rados.rados.0.smithi033.stdout:1728: writing smithi03385152-35 from 5316608 to 5947392 tid 2 2024-09-21T09:56:45.583 INFO:tasks.rados.rados.0.smithi033.stdout:1728: writing smithi03385152-35 from 5947392 to 6529024 tid 3 2024-09-21T09:56:45.586 INFO:tasks.rados.rados.0.smithi033.stdout:1728: writing smithi03385152-35 from 6529024 to 6922240 tid 4 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:1729: delete oid 4 current snap is 151 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:1726: finishing write tid 4 to smithi03385152-8 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:1726: finishing write tid 5 to smithi03385152-8 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:1726: finishing write tid 6 to smithi03385152-8 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 675 (ObjNum 560 snap 151 seq_num 560) dirty exists 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:1726: left oid 8 (ObjNum 560 snap 151 seq_num 560) 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:1728: finishing write tid 1 to smithi03385152-35 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:1728: finishing write tid 2 to smithi03385152-35 2024-09-21T09:56:45.587 INFO:tasks.rados.rados.0.smithi033.stdout:1728: finishing write tid 3 to smithi03385152-35 2024-09-21T09:56:45.595 INFO:tasks.rados.rados.0.smithi033.stdout:1726: done (2 left) 2024-09-21T09:56:45.595 INFO:tasks.rados.rados.0.smithi033.stdout:1729: done (1 left) 2024-09-21T09:56:45.595 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 151 2024-09-21T09:56:45.595 INFO:tasks.rados.rados.0.smithi033.stdout:1728: finishing write tid 4 to smithi03385152-35 2024-09-21T09:56:45.595 INFO:tasks.rados.rados.0.smithi033.stdout:1730: seq_num 564 ranges {6733824=655360,7389184=442368,7831552=598016,8429568=565248} 2024-09-21T09:56:45.635 INFO:tasks.rados.rados.0.smithi033.stdout:1730: writing smithi03385152-47 from 6733824 to 7389184 tid 1 2024-09-21T09:56:45.637 INFO:tasks.rados.rados.0.smithi033.stdout:1730: writing smithi03385152-47 from 7389184 to 7831552 tid 2 2024-09-21T09:56:45.641 INFO:tasks.rados.rados.0.smithi033.stdout:1730: writing smithi03385152-47 from 7831552 to 8429568 tid 3 2024-09-21T09:56:45.645 INFO:tasks.rados.rados.0.smithi033.stdout:1730: writing smithi03385152-47 from 8429568 to 8994816 tid 4 2024-09-21T09:56:45.645 INFO:tasks.rados.rados.0.smithi033.stdout:1731: rmattr oid 23 current snap is 151 2024-09-21T09:56:45.645 INFO:tasks.rados.rados.0.smithi033.stdout:1731: done (2 left) 2024-09-21T09:56:45.645 INFO:tasks.rados.rados.0.smithi033.stdout:1732: copy_from oid 30 from oid 2 current snap is 151 2024-09-21T09:56:45.645 INFO:tasks.rados.rados.0.smithi033.stdout:1728: finishing write tid 5 to smithi03385152-35 2024-09-21T09:56:45.645 INFO:tasks.rados.rados.0.smithi033.stdout:1728: finishing write tid 6 to smithi03385152-35 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 1095 (ObjNum 562 snap 151 seq_num 562) dirty exists 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:1728: left oid 35 (ObjNum 562 snap 151 seq_num 562) 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:1730: finishing write tid 1 to smithi03385152-47 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:1728: done (2 left) 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:1733: copy_from oid 18 from oid 44 current snap is 151 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:1730: finishing write tid 2 to smithi03385152-47 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:1730: finishing write tid 3 to smithi03385152-47 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 151 2024-09-21T09:56:45.646 INFO:tasks.rados.rados.0.smithi033.stdout:1734: seq_num 565 ranges {745472=671744,1417216=729088,2146304=753664,2899968=122880} 2024-09-21T09:56:45.653 INFO:tasks.rados.rados.0.smithi033.stdout:1734: writing smithi03385152-7 from 745472 to 1417216 tid 1 2024-09-21T09:56:45.658 INFO:tasks.rados.rados.0.smithi033.stdout:1734: writing smithi03385152-7 from 1417216 to 2146304 tid 2 2024-09-21T09:56:45.663 INFO:tasks.rados.rados.0.smithi033.stdout:1734: writing smithi03385152-7 from 2146304 to 2899968 tid 3 2024-09-21T09:56:45.664 INFO:tasks.rados.rados.0.smithi033.stdout:1734: writing smithi03385152-7 from 2899968 to 3022848 tid 4 2024-09-21T09:56:45.664 INFO:tasks.rados.rados.0.smithi033.stdout:1735: delete oid 46 current snap is 151 2024-09-21T09:56:45.664 INFO:tasks.rados.rados.0.smithi033.stdout:1730: finishing write tid 4 to smithi03385152-47 2024-09-21T09:56:45.664 INFO:tasks.rados.rados.0.smithi033.stdout:1730: finishing write tid 5 to smithi03385152-47 2024-09-21T09:56:45.664 INFO:tasks.rados.rados.0.smithi033.stdout:1730: finishing write tid 6 to smithi03385152-47 2024-09-21T09:56:45.664 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 655 (ObjNum 564 snap 151 seq_num 564) dirty exists 2024-09-21T09:56:45.665 INFO:tasks.rados.rados.0.smithi033.stdout:1730: left oid 47 (ObjNum 564 snap 151 seq_num 564) 2024-09-21T09:56:45.665 INFO:tasks.rados.rados.0.smithi033.stdout:1734: finishing write tid 1 to smithi03385152-7 2024-09-21T09:56:45.665 INFO:tasks.rados.rados.0.smithi033.stdout:1733: finishing copy_from racing read to smithi03385152-18 2024-09-21T09:56:45.665 INFO:tasks.rados.rados.0.smithi033.stdout:1733: finishing copy_from to smithi03385152-18 2024-09-21T09:56:45.666 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 998 (ObjNum 524 snap 142 seq_num 524) dirty exists 2024-09-21T09:56:45.667 INFO:tasks.rados.rados.0.smithi033.stdout:1734: finishing write tid 2 to smithi03385152-7 2024-09-21T09:56:45.668 INFO:tasks.rados.rados.0.smithi033.stdout:1734: finishing write tid 3 to smithi03385152-7 2024-09-21T09:56:45.668 INFO:tasks.rados.rados.0.smithi033.stdout:1734: finishing write tid 4 to smithi03385152-7 2024-09-21T09:56:45.668 INFO:tasks.rados.rados.0.smithi033.stdout:1734: finishing write tid 5 to smithi03385152-7 2024-09-21T09:56:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:1730: done (4 left) 2024-09-21T09:56:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:1733: done (3 left) 2024-09-21T09:56:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:1735: done (2 left) 2024-09-21T09:56:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:1736: read oid 16 snap -1 2024-09-21T09:56:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:1736: expect (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 151 2024-09-21T09:56:45.670 INFO:tasks.rados.rados.0.smithi033.stdout:1737: seq_num 567 ranges {3547136=704512,4251648=499712,4751360=770048,5521408=335872} 2024-09-21T09:56:45.681 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:56:45.681 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:56:45.681 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:56:45.681 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:56:45.681 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:56:45.681 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:56:45.682 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:56:45.682 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:56:45.682 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "7/20 daemons upgraded", 2024-09-21T09:56:45.682 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:56:45.682 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:56:45.682 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:56:45.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[96568]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:45.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[96568]: pgmap v195: 121 pgs: 121 active+clean; 146 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.4 MiB/s wr, 15 op/s 2024-09-21T09:56:45.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[96568]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:56:45.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[96568]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:45.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1447216909' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[98546]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:45.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[98546]: pgmap v195: 121 pgs: 121 active+clean; 146 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.4 MiB/s wr, 15 op/s 2024-09-21T09:56:45.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[98546]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:56:45.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[98546]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:45.683 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:45 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1447216909' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:45.694 INFO:tasks.rados.rados.0.smithi033.stdout:1737: writing smithi03385152-44 from 3547136 to 4251648 tid 1 2024-09-21T09:56:45.697 INFO:tasks.rados.rados.0.smithi033.stdout:1737: writing smithi03385152-44 from 4251648 to 4751360 tid 2 2024-09-21T09:56:45.702 INFO:tasks.rados.rados.0.smithi033.stdout:1737: writing smithi03385152-44 from 4751360 to 5521408 tid 3 2024-09-21T09:56:45.705 INFO:tasks.rados.rados.0.smithi033.stdout:1737: writing smithi03385152-44 from 5521408 to 5857280 tid 4 2024-09-21T09:56:45.705 INFO:tasks.rados.rados.0.smithi033.stdout:1738: snap_remove snap 150 2024-09-21T09:56:45.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:45.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:45.780 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:45.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:45 smithi105 ceph-mon[86649]: from='client.44204 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:45.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:45 smithi105 ceph-mon[86649]: pgmap v195: 121 pgs: 121 active+clean; 146 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 5.4 MiB/s wr, 15 op/s 2024-09-21T09:56:45.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:45 smithi105 ceph-mon[86649]: osdmap e378: 8 total, 8 up, 8 in 2024-09-21T09:56:45.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:45 smithi105 ceph-mon[86649]: from='client.44210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:45.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:45 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1447216909' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1738: done (4 left) 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1739: rollback oid 8 current snap is 151 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1739: done (4 left) 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1740: rollback oid 41 current snap is 151 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1740: done (4 left) 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1734: finishing write tid 6 to smithi03385152-7 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 1101 (ObjNum 565 snap 151 seq_num 565) dirty exists 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1734: left oid 7 (ObjNum 565 snap 151 seq_num 565) 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1741: read oid 8 snap -1 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1741: expect (ObjNum 560 snap 151 seq_num 560) 2024-09-21T09:56:46.510 INFO:tasks.rados.rados.0.smithi033.stdout:1734: done (4 left) 2024-09-21T09:56:46.511 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 151 2024-09-21T09:56:46.511 INFO:tasks.rados.rados.0.smithi033.stdout:1732: finishing copy_from racing read to smithi03385152-30 2024-09-21T09:56:46.511 INFO:tasks.rados.rados.0.smithi033.stdout:1737: finishing write tid 1 to smithi03385152-44 2024-09-21T09:56:46.511 INFO:tasks.rados.rados.0.smithi033.stdout:1737: finishing write tid 2 to smithi03385152-44 2024-09-21T09:56:46.511 INFO:tasks.rados.rados.0.smithi033.stdout:1742: seq_num 568 ranges {3776512=720896,4497408=786432,5283840=581632,5865472=237568} 2024-09-21T09:56:46.535 INFO:tasks.rados.rados.0.smithi033.stdout:1742: writing smithi03385152-33 from 3776512 to 4497408 tid 1 2024-09-21T09:56:46.540 INFO:tasks.rados.rados.0.smithi033.stdout:1742: writing smithi03385152-33 from 4497408 to 5283840 tid 2 2024-09-21T09:56:46.544 INFO:tasks.rados.rados.0.smithi033.stdout:1742: writing smithi03385152-33 from 5283840 to 5865472 tid 3 2024-09-21T09:56:46.546 INFO:tasks.rados.rados.0.smithi033.stdout:1742: writing smithi03385152-33 from 5865472 to 6103040 tid 4 2024-09-21T09:56:46.546 INFO:tasks.rados.rados.0.smithi033.stdout:1743: snap_create 2024-09-21T09:56:46.546 INFO:tasks.rados.rados.0.smithi033.stdout:1737: finishing write tid 3 to smithi03385152-44 2024-09-21T09:56:46.546 INFO:tasks.rados.rados.0.smithi033.stdout:1737: finishing write tid 4 to smithi03385152-44 2024-09-21T09:56:46.546 INFO:tasks.rados.rados.0.smithi033.stdout:1737: finishing write tid 5 to smithi03385152-44 2024-09-21T09:56:46.546 INFO:tasks.rados.rados.0.smithi033.stdout:1737: finishing write tid 6 to smithi03385152-44 2024-09-21T09:56:46.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1007 (ObjNum 567 snap 151 seq_num 567) dirty exists 2024-09-21T09:56:46.547 INFO:tasks.rados.rados.0.smithi033.stdout:1737: left oid 44 (ObjNum 567 snap 151 seq_num 567) 2024-09-21T09:56:46.547 INFO:tasks.rados.rados.0.smithi033.stdout:1736: expect (ObjNum 507 snap 135 seq_num 507) 2024-09-21T09:56:46.649 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:46 smithi033 systemd[1]: Stopping Ceph osd.2 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:56:46.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[98546]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[98546]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[98546]: from='client.44228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[96568]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[96568]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[96568]: from='client.44228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:46.650 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:46 smithi105 ceph-mon[86649]: from='client.44216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:46 smithi105 ceph-mon[86649]: osdmap e379: 8 total, 8 up, 8 in 2024-09-21T09:56:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:46 smithi105 ceph-mon[86649]: from='client.44228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:56:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:46.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:56:46.996 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[55754]: 2024-09-21T09:56:46.646+0000 7ff4e70fe640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:56:46.996 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[55754]: 2024-09-21T09:56:46.646+0000 7ff4e70fe640 -1 osd.2 380 *** Got signal Terminated *** 2024-09-21T09:56:46.996 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[55754]: 2024-09-21T09:56:46.646+0000 7ff4e70fe640 -1 osd.2 380 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1732: finishing copy_from to smithi03385152-30 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 885 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1742: finishing write tid 1 to smithi03385152-33 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1742: finishing write tid 2 to smithi03385152-33 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1742: finishing write tid 3 to smithi03385152-33 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1742: finishing write tid 4 to smithi03385152-33 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1742: finishing write tid 5 to smithi03385152-33 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1742: finishing write tid 6 to smithi03385152-33 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1031 (ObjNum 568 snap 151 seq_num 568) dirty exists 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1742: left oid 33 (ObjNum 568 snap 151 seq_num 568) 2024-09-21T09:56:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:1741: expect (ObjNum 560 snap 151 seq_num 560) 2024-09-21T09:56:47.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:47 smithi105 ceph-mon[86649]: pgmap v198: 121 pgs: 121 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 43 op/s 2024-09-21T09:56:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:47 smithi105 ceph-mon[86649]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:56:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:47 smithi105 ceph-mon[86649]: osd.2 marked itself down and dead 2024-09-21T09:56:47.813 INFO:tasks.rados.rados.0.smithi033.stdout:1732: done (5 left) 2024-09-21T09:56:47.813 INFO:tasks.rados.rados.0.smithi033.stdout:1736: done (4 left) 2024-09-21T09:56:47.813 INFO:tasks.rados.rados.0.smithi033.stdout:1737: done (3 left) 2024-09-21T09:56:47.813 INFO:tasks.rados.rados.0.smithi033.stdout:1741: done (2 left) 2024-09-21T09:56:47.813 INFO:tasks.rados.rados.0.smithi033.stdout:1742: done (1 left) 2024-09-21T09:56:47.813 INFO:tasks.rados.rados.0.smithi033.stdout:1743: done (0 left) 2024-09-21T09:56:47.813 INFO:tasks.rados.rados.0.smithi033.stdout:1744: delete oid 40 current snap is 152 2024-09-21T09:56:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:47 smithi033 ceph-mon[96568]: pgmap v198: 121 pgs: 121 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 43 op/s 2024-09-21T09:56:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:47 smithi033 ceph-mon[96568]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:56:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:47 smithi033 ceph-mon[96568]: osd.2 marked itself down and dead 2024-09-21T09:56:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:47 smithi033 ceph-mon[98546]: pgmap v198: 121 pgs: 121 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 43 op/s 2024-09-21T09:56:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:47 smithi033 ceph-mon[98546]: osdmap e380: 8 total, 8 up, 8 in 2024-09-21T09:56:47.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:47 smithi033 ceph-mon[98546]: osd.2 marked itself down and dead 2024-09-21T09:56:48.324 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:48 smithi033 podman[112876]: 2024-09-21 09:56:48.062184589 +0000 UTC m=+1.545274514 container died 56b187b7ae4c88fc4340c7113854af5d7db849722ef4c09a7e620d1bb8e38895 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T09:56:48.474 INFO:tasks.rados.rados.0.smithi033.stdout:1744: done (0 left) 2024-09-21T09:56:48.474 INFO:tasks.rados.rados.0.smithi033.stdout:1745: read oid 17 snap 151 2024-09-21T09:56:48.474 INFO:tasks.rados.rados.0.smithi033.stdout:1745: expect (ObjNum 541 snap 145 seq_num 541) 2024-09-21T09:56:48.474 INFO:tasks.rados.rados.0.smithi033.stdout:1746: rollback oid 11 current snap is 152 2024-09-21T09:56:48.474 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 11 to 151 2024-09-21T09:56:48.474 INFO:tasks.rados.rados.0.smithi033.stdout:1747: snap_create 2024-09-21T09:56:48.477 INFO:tasks.rados.rados.0.smithi033.stdout:1746: finishing rollback tid 0 to smithi03385152-11 2024-09-21T09:56:48.478 INFO:tasks.rados.rados.0.smithi033.stdout:1746: finishing rollback tid 1 to smithi03385152-11 2024-09-21T09:56:48.478 INFO:tasks.rados.rados.0.smithi033.stdout:1746: finishing rollback tid 2 to smithi03385152-11 2024-09-21T09:56:48.478 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 725 (ObjNum 555 snap 150 seq_num 555) dirty exists 2024-09-21T09:56:48.517 INFO:tasks.rados.rados.0.smithi033.stdout:1745: expect (ObjNum 541 snap 145 seq_num 541) 2024-09-21T09:56:48.722 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:48 smithi033 ceph-mon[98546]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:56:48.723 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:48 smithi033 ceph-mon[98546]: osdmap e381: 8 total, 7 up, 8 in 2024-09-21T09:56:48.723 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:48 smithi033 ceph-mon[98546]: osdmap e382: 8 total, 7 up, 8 in 2024-09-21T09:56:48.723 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:48 smithi033 podman[112876]: 2024-09-21 09:56:48.420059781 +0000 UTC m=+1.903149715 container cleanup 56b187b7ae4c88fc4340c7113854af5d7db849722ef4c09a7e620d1bb8e38895 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:48.723 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:48 smithi033 bash[112876]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2 2024-09-21T09:56:48.723 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:48 smithi033 ceph-mon[96568]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:56:48.723 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:48 smithi033 ceph-mon[96568]: osdmap e381: 8 total, 7 up, 8 in 2024-09-21T09:56:48.723 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:48 smithi033 ceph-mon[96568]: osdmap e382: 8 total, 7 up, 8 in 2024-09-21T09:56:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:48 smithi105 ceph-mon[86649]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:56:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:48 smithi105 ceph-mon[86649]: osdmap e381: 8 total, 7 up, 8 in 2024-09-21T09:56:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:48 smithi105 ceph-mon[86649]: osdmap e382: 8 total, 7 up, 8 in 2024-09-21T09:56:48.996 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:48 smithi033 podman[112896]: 2024-09-21 09:56:48.721312355 +0000 UTC m=+0.655711117 container remove 56b187b7ae4c88fc4340c7113854af5d7db849722ef4c09a7e620d1bb8e38895 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=quincy-fbd1a6a, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:56:49.335 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:49 smithi033 podman[112987]: 2024-09-21 09:56:48.974464169 +0000 UTC m=+0.016912661 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:56:49.335 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:49 smithi033 podman[112987]: 2024-09-21 09:56:49.08058703 +0000 UTC m=+0.123035506 container create 7cccc4a7b3fab264385410ed6f554c4d7731fc34047193d6e7b9a58fda270321 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T09:56:49.464 INFO:tasks.rados.rados.0.smithi033.stdout:1745: done (2 left) 2024-09-21T09:56:49.464 INFO:tasks.rados.rados.0.smithi033.stdout:1746: done (1 left) 2024-09-21T09:56:49.464 INFO:tasks.rados.rados.0.smithi033.stdout:1747: done (0 left) 2024-09-21T09:56:49.464 INFO:tasks.rados.rados.0.smithi033.stdout:1748: setattr oid 14 current snap is 153 2024-09-21T09:56:49.465 INFO:tasks.rados.rados.0.smithi033.stdout:1749: snap_create 2024-09-21T09:56:49.471 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 917 (ObjNum 551 snap 149 seq_num 551) dirty exists 2024-09-21T09:56:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:49 smithi033 ceph-mon[98546]: pgmap v201: 121 pgs: 13 stale+active+clean, 108 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 43 op/s 2024-09-21T09:56:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:49 smithi033 ceph-mon[98546]: osdmap e383: 8 total, 7 up, 8 in 2024-09-21T09:56:49.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:49 smithi033 ceph-mon[96568]: pgmap v201: 121 pgs: 13 stale+active+clean, 108 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 43 op/s 2024-09-21T09:56:49.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:49 smithi033 ceph-mon[96568]: osdmap e383: 8 total, 7 up, 8 in 2024-09-21T09:56:49.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:49 smithi033 podman[112987]: 2024-09-21 09:56:49.333656465 +0000 UTC m=+0.376104957 container init 7cccc4a7b3fab264385410ed6f554c4d7731fc34047193d6e7b9a58fda270321 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True) 2024-09-21T09:56:49.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:49 smithi033 podman[112987]: 2024-09-21 09:56:49.338926724 +0000 UTC m=+0.381375200 container start 7cccc4a7b3fab264385410ed6f554c4d7731fc34047193d6e7b9a58fda270321 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:49.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:49 smithi033 podman[112987]: 2024-09-21 09:56:49.416757798 +0000 UTC m=+0.459206275 container attach 7cccc4a7b3fab264385410ed6f554c4d7731fc34047193d6e7b9a58fda270321 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, RELEASE=main-62da54b) 2024-09-21T09:56:49.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:49 smithi033 podman[112987]: 2024-09-21 09:56:49.540387664 +0000 UTC m=+0.582836147 container died 7cccc4a7b3fab264385410ed6f554c4d7731fc34047193d6e7b9a58fda270321 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux ) 2024-09-21T09:56:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:49 smithi105 ceph-mon[86649]: pgmap v201: 121 pgs: 13 stale+active+clean, 108 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 11 MiB/s wr, 43 op/s 2024-09-21T09:56:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:49 smithi105 ceph-mon[86649]: osdmap e383: 8 total, 7 up, 8 in 2024-09-21T09:56:50.099 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 podman[113030]: 2024-09-21 09:56:50.09521349 +0000 UTC m=+0.548582753 container remove 7cccc4a7b3fab264385410ed6f554c4d7731fc34047193d6e7b9a58fda270321 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T09:56:50.466 INFO:tasks.rados.rados.0.smithi033.stdout:1748: done (1 left) 2024-09-21T09:56:50.467 INFO:tasks.rados.rados.0.smithi033.stdout:1749: done (0 left) 2024-09-21T09:56:50.467 INFO:tasks.rados.rados.0.smithi033.stdout:1750: delete oid 24 current snap is 154 2024-09-21T09:56:50.470 INFO:tasks.rados.rados.0.smithi033.stdout:1750: done (0 left) 2024-09-21T09:56:50.470 INFO:tasks.rados.rados.0.smithi033.stdout:1751: rmattr oid 14 current snap is 154 2024-09-21T09:56:50.470 INFO:tasks.rados.rados.0.smithi033.stdout:1752: delete oid 7 current snap is 154 2024-09-21T09:56:50.476 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 919 (ObjNum 551 snap 149 seq_num 551) dirty exists 2024-09-21T09:56:50.491 INFO:tasks.rados.rados.0.smithi033.stdout:1751: done (1 left) 2024-09-21T09:56:50.491 INFO:tasks.rados.rados.0.smithi033.stdout:1752: done (0 left) 2024-09-21T09:56:50.491 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 154 2024-09-21T09:56:50.491 INFO:tasks.rados.rados.0.smithi033.stdout:1753: seq_num 572 ranges {0=786432,786432=770048,1556480=557056,2113536=278528} 2024-09-21T09:56:50.495 INFO:tasks.rados.rados.0.smithi033.stdout:1753: writing smithi03385152-4 from 0 to 786432 tid 1 2024-09-21T09:56:50.496 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.2.service: Deactivated successfully. 2024-09-21T09:56:50.496 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 systemd[1]: Stopped Ceph osd.2 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:56:50.496 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.2.service: Consumed 13.503s CPU time. 2024-09-21T09:56:50.496 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 systemd[1]: Starting Ceph osd.2 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:56:50.500 INFO:tasks.rados.rados.0.smithi033.stdout:1753: writing smithi03385152-4 from 786432 to 1556480 tid 2 2024-09-21T09:56:50.504 INFO:tasks.rados.rados.0.smithi033.stdout:1753: writing smithi03385152-4 from 1556480 to 2113536 tid 3 2024-09-21T09:56:50.506 INFO:tasks.rados.rados.0.smithi033.stdout:1753: writing smithi03385152-4 from 2113536 to 2392064 tid 4 2024-09-21T09:56:50.506 INFO:tasks.rados.rados.0.smithi033.stdout:1754: snap_create 2024-09-21T09:56:50.506 INFO:tasks.rados.rados.0.smithi033.stdout:1753: finishing write tid 1 to smithi03385152-4 2024-09-21T09:56:50.506 INFO:tasks.rados.rados.0.smithi033.stdout:1753: finishing write tid 2 to smithi03385152-4 2024-09-21T09:56:50.507 INFO:tasks.rados.rados.0.smithi033.stdout:1753: finishing write tid 3 to smithi03385152-4 2024-09-21T09:56:50.508 INFO:tasks.rados.rados.0.smithi033.stdout:1753: finishing write tid 4 to smithi03385152-4 2024-09-21T09:56:50.509 INFO:tasks.rados.rados.0.smithi033.stdout:1753: finishing write tid 5 to smithi03385152-4 2024-09-21T09:56:50.510 INFO:tasks.rados.rados.0.smithi033.stdout:1753: finishing write tid 6 to smithi03385152-4 2024-09-21T09:56:50.510 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1012 (ObjNum 572 snap 154 seq_num 572) dirty exists 2024-09-21T09:56:50.510 INFO:tasks.rados.rados.0.smithi033.stdout:1753: left oid 4 (ObjNum 572 snap 154 seq_num 572) 2024-09-21T09:56:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:50.850 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 podman[113135]: 2024-09-21 09:56:50.460558366 +0000 UTC m=+0.017106557 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:56:50.850 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 podman[113135]: 2024-09-21 09:56:50.646236695 +0000 UTC m=+0.202784877 container create 43dbdbad10b1e091442e13fb4dfbd3f78ca2ef75d80a6294f1d16e68deaf558a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:51.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 podman[113135]: 2024-09-21 09:56:50.849115695 +0000 UTC m=+0.405663883 container init 43dbdbad10b1e091442e13fb4dfbd3f78ca2ef75d80a6294f1d16e68deaf558a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_BRANCH=HEAD) 2024-09-21T09:56:51.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 podman[113135]: 2024-09-21 09:56:50.853953876 +0000 UTC m=+0.410502059 container start 43dbdbad10b1e091442e13fb4dfbd3f78ca2ef75d80a6294f1d16e68deaf558a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T09:56:51.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 podman[113135]: 2024-09-21 09:56:50.948880972 +0000 UTC m=+0.505429162 container attach 43dbdbad10b1e091442e13fb4dfbd3f78ca2ef75d80a6294f1d16e68deaf558a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T09:56:51.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:56:51.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:50 smithi033 bash[113135]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:56:51.247 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:56:51.247 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 bash[113135]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:56:51.471 INFO:tasks.rados.rados.0.smithi033.stdout:1753: done (1 left) 2024-09-21T09:56:51.471 INFO:tasks.rados.rados.0.smithi033.stdout:1754: done (0 left) 2024-09-21T09:56:51.472 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 155 2024-09-21T09:56:51.472 INFO:tasks.rados.rados.0.smithi033.stdout:1755: seq_num 573 ranges {0=802816,802816=655360,1458176=720896,2179072=229376} 2024-09-21T09:56:51.476 INFO:tasks.rados.rados.0.smithi033.stdout:1755: writing smithi03385152-45 from 0 to 802816 tid 1 2024-09-21T09:56:51.479 INFO:tasks.rados.rados.0.smithi033.stdout:1755: writing smithi03385152-45 from 802816 to 1458176 tid 2 2024-09-21T09:56:51.484 INFO:tasks.rados.rados.0.smithi033.stdout:1755: writing smithi03385152-45 from 1458176 to 2179072 tid 3 2024-09-21T09:56:51.485 INFO:tasks.rados.rados.0.smithi033.stdout:1755: writing smithi03385152-45 from 2179072 to 2408448 tid 4 2024-09-21T09:56:51.485 INFO:tasks.rados.rados.0.smithi033.stdout:1756: snap_remove snap 152 2024-09-21T09:56:51.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:56:51] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.51.0" 2024-09-21T09:56:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-mon[98546]: pgmap v204: 121 pgs: 4 active+undersized, 12 stale+active+clean, 1 active+undersized+degraded, 104 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 4/222 objects degraded (1.802%) 2024-09-21T09:56:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-mon[98546]: osdmap e384: 8 total, 7 up, 8 in 2024-09-21T09:56:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-mon[98546]: Health check failed: Degraded data redundancy: 4/222 objects degraded (1.802%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:56:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-mon[96568]: pgmap v204: 121 pgs: 4 active+undersized, 12 stale+active+clean, 1 active+undersized+degraded, 104 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 4/222 objects degraded (1.802%) 2024-09-21T09:56:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-mon[96568]: osdmap e384: 8 total, 7 up, 8 in 2024-09-21T09:56:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-mon[96568]: Health check failed: Degraded data redundancy: 4/222 objects degraded (1.802%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:56:51.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:56:51.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 bash[113135]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:56:51.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:56:51.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 bash[113135]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:56:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:51 smithi105 ceph-mon[86649]: pgmap v204: 121 pgs: 4 active+undersized, 12 stale+active+clean, 1 active+undersized+degraded, 104 active+clean; 167 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 4/222 objects degraded (1.802%) 2024-09-21T09:56:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:51 smithi105 ceph-mon[86649]: osdmap e384: 8 total, 7 up, 8 in 2024-09-21T09:56:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:51 smithi105 ceph-mon[86649]: Health check failed: Degraded data redundancy: 4/222 objects degraded (1.802%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:56:52.104 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 bash[113135]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 bash[113135]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 bash[113135]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 bash[113135]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate[113150]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 bash[113135]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-21T09:56:52.105 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:51 smithi033 podman[113135]: 2024-09-21 09:56:51.792214707 +0000 UTC m=+1.348762889 container died 43dbdbad10b1e091442e13fb4dfbd3f78ca2ef75d80a6294f1d16e68deaf558a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:52.476 INFO:tasks.rados.rados.0.smithi033.stdout:1756: done (1 left) 2024-09-21T09:56:52.476 INFO:tasks.rados.rados.0.smithi033.stdout:1757: copy_from oid 25 from oid 40 current snap is 155 2024-09-21T09:56:52.476 INFO:tasks.rados.rados.0.smithi033.stdout:1755: finishing write tid 1 to smithi03385152-45 2024-09-21T09:56:52.476 INFO:tasks.rados.rados.0.smithi033.stdout:1755: finishing write tid 2 to smithi03385152-45 2024-09-21T09:56:52.476 INFO:tasks.rados.rados.0.smithi033.stdout:1755: finishing write tid 3 to smithi03385152-45 2024-09-21T09:56:52.476 INFO:tasks.rados.rados.0.smithi033.stdout:1755: finishing write tid 4 to smithi03385152-45 2024-09-21T09:56:52.477 INFO:tasks.rados.rados.0.smithi033.stdout:1755: finishing write tid 5 to smithi03385152-45 2024-09-21T09:56:52.477 INFO:tasks.rados.rados.0.smithi033.stdout:1755: finishing write tid 6 to smithi03385152-45 2024-09-21T09:56:52.477 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 924 (ObjNum 573 snap 155 seq_num 573) dirty exists 2024-09-21T09:56:52.477 INFO:tasks.rados.rados.0.smithi033.stdout:1755: left oid 45 (ObjNum 573 snap 155 seq_num 573) 2024-09-21T09:56:52.477 INFO:tasks.rados.rados.0.smithi033.stdout:1755: done (1 left) 2024-09-21T09:56:52.477 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 155 2024-09-21T09:56:52.477 INFO:tasks.rados.rados.0.smithi033.stdout:1758: seq_num 574 ranges {1851392=16384} 2024-09-21T09:56:52.486 INFO:tasks.rados.rados.0.smithi033.stdout:1758: writing smithi03385152-32 from 1851392 to 1867776 tid 1 2024-09-21T09:56:52.487 INFO:tasks.rados.rados.0.smithi033.stdout:1759: snap_remove snap 151 2024-09-21T09:56:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:52 smithi033 ceph-mon[98546]: osdmap e385: 8 total, 7 up, 8 in 2024-09-21T09:56:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:52 smithi033 ceph-mon[96568]: osdmap e385: 8 total, 7 up, 8 in 2024-09-21T09:56:52.746 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:52 smithi033 podman[113313]: 2024-09-21 09:56:52.44446831 +0000 UTC m=+0.645808161 container remove 43dbdbad10b1e091442e13fb4dfbd3f78ca2ef75d80a6294f1d16e68deaf558a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-activate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:56:52.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:52 smithi033 podman[113377]: 2024-09-21 09:56:52.572752423 +0000 UTC m=+0.017257276 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:56:52.747 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:52 smithi033 podman[113377]: 2024-09-21 09:56:52.67020794 +0000 UTC m=+0.114712788 container create 76dbf29565ba59e70f9e81249324f2e546ec2c319fb6e16c10788f09ee8007ff (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, io.buildah.version=1.37.2) 2024-09-21T09:56:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:52 smithi105 ceph-mon[86649]: osdmap e385: 8 total, 7 up, 8 in 2024-09-21T09:56:53.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:52 smithi033 podman[113377]: 2024-09-21 09:56:52.88094217 +0000 UTC m=+0.325447021 container init 76dbf29565ba59e70f9e81249324f2e546ec2c319fb6e16c10788f09ee8007ff (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2) 2024-09-21T09:56:53.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:52 smithi033 podman[113377]: 2024-09-21 09:56:52.886210077 +0000 UTC m=+0.330714918 container start 76dbf29565ba59e70f9e81249324f2e546ec2c319fb6e16c10788f09ee8007ff (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:56:53.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:52 smithi033 ceph-osd[113415]: -- 172.21.15.33:0/3751799470 <== mon.2 v2:172.21.15.33:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559076c85a40 con 0x559077a58400 2024-09-21T09:56:53.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:52 smithi033 bash[113377]: 76dbf29565ba59e70f9e81249324f2e546ec2c319fb6e16c10788f09ee8007ff 2024-09-21T09:56:53.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:52 smithi033 systemd[1]: Started Ceph osd.2 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1759: done (2 left) 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1760: delete oid 19 current snap is 155 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1757: finishing copy_from to smithi03385152-25 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1757: got expected ENOENT (src dne) 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1757: finishing copy_from racing read to smithi03385152-25 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1758: finishing write tid 1 to smithi03385152-32 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1758: finishing write tid 2 to smithi03385152-32 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1758: finishing write tid 3 to smithi03385152-32 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 169 (ObjNum 574 snap 155 seq_num 574) dirty exists 2024-09-21T09:56:53.467 INFO:tasks.rados.rados.0.smithi033.stdout:1758: left oid 32 (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:56:53.470 INFO:tasks.rados.rados.0.smithi033.stdout:1757: done (2 left) 2024-09-21T09:56:53.470 INFO:tasks.rados.rados.0.smithi033.stdout:1758: done (1 left) 2024-09-21T09:56:53.470 INFO:tasks.rados.rados.0.smithi033.stdout:1760: done (0 left) 2024-09-21T09:56:53.470 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 155 2024-09-21T09:56:53.471 INFO:tasks.rados.rados.0.smithi033.stdout:1761: seq_num 576 ranges {1925120=49152} 2024-09-21T09:56:53.481 INFO:tasks.rados.rados.0.smithi033.stdout:1761: writing smithi03385152-23 from 1925120 to 1974272 tid 1 2024-09-21T09:56:53.482 INFO:tasks.rados.rados.0.smithi033.stdout:1762: snap_create 2024-09-21T09:56:53.488 INFO:tasks.rados.rados.0.smithi033.stdout:1761: finishing write tid 1 to smithi03385152-23 2024-09-21T09:56:53.488 INFO:tasks.rados.rados.0.smithi033.stdout:1761: finishing write tid 2 to smithi03385152-23 2024-09-21T09:56:53.489 INFO:tasks.rados.rados.0.smithi033.stdout:1761: finishing write tid 3 to smithi03385152-23 2024-09-21T09:56:53.489 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 709 (ObjNum 576 snap 155 seq_num 576) dirty exists 2024-09-21T09:56:53.489 INFO:tasks.rados.rados.0.smithi033.stdout:1761: left oid 23 (ObjNum 576 snap 155 seq_num 576) 2024-09-21T09:56:53.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[96568]: pgmap v207: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 162 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:56:53.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[96568]: osdmap e386: 8 total, 7 up, 8 in 2024-09-21T09:56:53.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:53.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:53.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:53.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[96568]: osdmap e387: 8 total, 7 up, 8 in 2024-09-21T09:56:53.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[98546]: pgmap v207: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 162 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:56:53.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[98546]: osdmap e386: 8 total, 7 up, 8 in 2024-09-21T09:56:53.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:53.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:53.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:53.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:53 smithi033 ceph-mon[98546]: osdmap e387: 8 total, 7 up, 8 in 2024-09-21T09:56:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:53 smithi105 ceph-mon[86649]: pgmap v207: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 162 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 13 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:56:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:53 smithi105 ceph-mon[86649]: osdmap e386: 8 total, 7 up, 8 in 2024-09-21T09:56:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:56:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:53 smithi105 ceph-mon[86649]: osdmap e387: 8 total, 7 up, 8 in 2024-09-21T09:56:54.482 INFO:tasks.rados.rados.0.smithi033.stdout:1761: done (1 left) 2024-09-21T09:56:54.482 INFO:tasks.rados.rados.0.smithi033.stdout:1762: done (0 left) 2024-09-21T09:56:54.483 INFO:tasks.rados.rados.0.smithi033.stdout:1763: read oid 33 snap -1 2024-09-21T09:56:54.483 INFO:tasks.rados.rados.0.smithi033.stdout:1763: expect (ObjNum 568 snap 151 seq_num 568) 2024-09-21T09:56:54.483 INFO:tasks.rados.rados.0.smithi033.stdout:1764: read oid 6 snap -1 2024-09-21T09:56:54.483 INFO:tasks.rados.rados.0.smithi033.stdout:1764: expect (ObjNum 550 snap 149 seq_num 550) 2024-09-21T09:56:54.483 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 156 2024-09-21T09:56:54.483 INFO:tasks.rados.rados.0.smithi033.stdout:1765: seq_num 577 ranges {9404416=65536} 2024-09-21T09:56:54.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.2\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.2\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:54.536 INFO:tasks.rados.rados.0.smithi033.stdout:1765: writing smithi03385152-2 from 9404416 to 9469952 tid 1 2024-09-21T09:56:54.536 INFO:tasks.rados.rados.0.smithi033.stdout:1766: delete oid 8 current snap is 156 2024-09-21T09:56:54.536 INFO:tasks.rados.rados.0.smithi033.stdout:1763: expect (ObjNum 568 snap 151 seq_num 568) 2024-09-21T09:56:54.785 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:54 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[113392]: 2024-09-21T09:56:54.520+0000 7fe7b71af640 -1 Falling back to public interface 2024-09-21T09:56:54.788 INFO:tasks.rados.rados.0.smithi033.stdout:1764: expect (ObjNum 550 snap 149 seq_num 550) 2024-09-21T09:56:54.912 INFO:tasks.rados.rados.0.smithi033.stdout:1765: finishing write tid 1 to smithi03385152-2 2024-09-21T09:56:54.912 INFO:tasks.rados.rados.0.smithi033.stdout:1765: finishing write tid 2 to smithi03385152-2 2024-09-21T09:56:54.912 INFO:tasks.rados.rados.0.smithi033.stdout:1765: finishing write tid 3 to smithi03385152-2 2024-09-21T09:56:54.912 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 887 (ObjNum 577 snap 156 seq_num 577) dirty exists 2024-09-21T09:56:54.912 INFO:tasks.rados.rados.0.smithi033.stdout:1765: left oid 2 (ObjNum 577 snap 156 seq_num 577) 2024-09-21T09:56:54.912 INFO:tasks.rados.rados.0.smithi033.stdout:1763: done (3 left) 2024-09-21T09:56:54.913 INFO:tasks.rados.rados.0.smithi033.stdout:1764: done (2 left) 2024-09-21T09:56:54.913 INFO:tasks.rados.rados.0.smithi033.stdout:1765: done (1 left) 2024-09-21T09:56:54.913 INFO:tasks.rados.rados.0.smithi033.stdout:1766: done (0 left) 2024-09-21T09:56:54.913 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 156 2024-09-21T09:56:54.913 INFO:tasks.rados.rados.0.smithi033.stdout:1767: seq_num 579 ranges {1908736=98304} 2024-09-21T09:56:54.923 INFO:tasks.rados.rados.0.smithi033.stdout:1767: writing smithi03385152-42 from 1908736 to 2007040 tid 1 2024-09-21T09:56:54.923 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 156 2024-09-21T09:56:54.923 INFO:tasks.rados.rados.0.smithi033.stdout:1768: seq_num 580 ranges {0=114688} 2024-09-21T09:56:54.923 INFO:tasks.rados.rados.0.smithi033.stdout:1768: writing smithi03385152-39 from 0 to 114688 tid 1 2024-09-21T09:56:54.924 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 156 2024-09-21T09:56:54.924 INFO:tasks.rados.rados.0.smithi033.stdout:1769: seq_num 581 ranges {1974272=139264} 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1769: writing smithi03385152-23 from 1974272 to 2113536 tid 1 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1770: delete oid 13 current snap is 156 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1767: finishing write tid 1 to smithi03385152-42 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1768: finishing write tid 1 to smithi03385152-39 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1768: finishing write tid 2 to smithi03385152-39 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1767: finishing write tid 2 to smithi03385152-42 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1767: finishing write tid 3 to smithi03385152-42 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 739 (ObjNum 579 snap 156 seq_num 579) dirty exists 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1767: left oid 42 (ObjNum 579 snap 156 seq_num 579) 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1768: finishing write tid 3 to smithi03385152-39 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 233 (ObjNum 580 snap 156 seq_num 580) dirty exists 2024-09-21T09:56:54.936 INFO:tasks.rados.rados.0.smithi033.stdout:1768: left oid 39 (ObjNum 580 snap 156 seq_num 580) 2024-09-21T09:56:54.937 INFO:tasks.rados.rados.0.smithi033.stdout:1769: finishing write tid 1 to smithi03385152-23 2024-09-21T09:56:54.938 INFO:tasks.rados.rados.0.smithi033.stdout:1769: finishing write tid 2 to smithi03385152-23 2024-09-21T09:56:54.938 INFO:tasks.rados.rados.0.smithi033.stdout:1767: done (3 left) 2024-09-21T09:56:54.939 INFO:tasks.rados.rados.0.smithi033.stdout:1768: done (2 left) 2024-09-21T09:56:54.939 INFO:tasks.rados.rados.0.smithi033.stdout:1770: done (1 left) 2024-09-21T09:56:54.939 INFO:tasks.rados.rados.0.smithi033.stdout:1771: rmattr oid 7 current snap is 156 2024-09-21T09:56:54.939 INFO:tasks.rados.rados.0.smithi033.stdout:1771: done (1 left) 2024-09-21T09:56:54.939 INFO:tasks.rados.rados.0.smithi033.stdout:1772: read oid 29 snap -1 2024-09-21T09:56:54.939 INFO:tasks.rados.rados.0.smithi033.stdout:1772: expect deleted 2024-09-21T09:56:54.939 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 156 2024-09-21T09:56:54.939 INFO:tasks.rados.rados.0.smithi033.stdout:1773: seq_num 583 ranges {0=172032} 2024-09-21T09:56:54.940 INFO:tasks.rados.rados.0.smithi033.stdout:1773: writing smithi03385152-7 from 0 to 172032 tid 1 2024-09-21T09:56:54.940 INFO:tasks.rados.rados.0.smithi033.stdout:1774: read oid 13 snap -1 2024-09-21T09:56:54.940 INFO:tasks.rados.rados.0.smithi033.stdout:1774: expect deleted 2024-09-21T09:56:54.940 INFO:tasks.rados.rados.0.smithi033.stdout:1775: read oid 26 snap 155 2024-09-21T09:56:54.940 INFO:tasks.rados.rados.0.smithi033.stdout:1775: expect (ObjNum 554 snap 149 seq_num 554) 2024-09-21T09:56:54.940 INFO:tasks.rados.rados.0.smithi033.stdout:1776: delete oid 43 current snap is 156 2024-09-21T09:56:54.941 INFO:tasks.rados.rados.0.smithi033.stdout:1769: finishing write tid 3 to smithi03385152-23 2024-09-21T09:56:54.941 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 711 (ObjNum 581 snap 156 seq_num 581) dirty exists 2024-09-21T09:56:54.941 INFO:tasks.rados.rados.0.smithi033.stdout:1769: left oid 23 (ObjNum 581 snap 156 seq_num 581) 2024-09-21T09:56:54.943 INFO:tasks.rados.rados.0.smithi033.stdout:1769: done (5 left) 2024-09-21T09:56:54.943 INFO:tasks.rados.rados.0.smithi033.stdout:1772: done (4 left) 2024-09-21T09:56:54.943 INFO:tasks.rados.rados.0.smithi033.stdout:1774: done (3 left) 2024-09-21T09:56:54.943 INFO:tasks.rados.rados.0.smithi033.stdout:1776: done (2 left) 2024-09-21T09:56:54.943 INFO:tasks.rados.rados.0.smithi033.stdout:1777: rollback oid 36 current snap is 156 2024-09-21T09:56:54.943 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 155 2024-09-21T09:56:54.943 INFO:tasks.rados.rados.0.smithi033.stdout:1773: finishing write tid 1 to smithi03385152-7 2024-09-21T09:56:54.944 INFO:tasks.rados.rados.0.smithi033.stdout:1778: snap_create 2024-09-21T09:56:54.944 INFO:tasks.rados.rados.0.smithi033.stdout:1773: finishing write tid 2 to smithi03385152-7 2024-09-21T09:56:54.946 INFO:tasks.rados.rados.0.smithi033.stdout:1773: finishing write tid 3 to smithi03385152-7 2024-09-21T09:56:54.946 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 1119 (ObjNum 583 snap 156 seq_num 583) dirty exists 2024-09-21T09:56:54.946 INFO:tasks.rados.rados.0.smithi033.stdout:1773: left oid 7 (ObjNum 583 snap 156 seq_num 583) 2024-09-21T09:56:54.947 INFO:tasks.rados.rados.0.smithi033.stdout:1777: finishing rollback tid 0 to smithi03385152-36 2024-09-21T09:56:54.948 INFO:tasks.rados.rados.0.smithi033.stdout:1777: finishing rollback tid 1 to smithi03385152-36 2024-09-21T09:56:54.948 INFO:tasks.rados.rados.0.smithi033.stdout:1777: finishing rollback tid 2 to smithi03385152-36 2024-09-21T09:56:54.948 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1123 (ObjNum 546 snap 147 seq_num 546) dirty exists 2024-09-21T09:56:54.960 INFO:tasks.rados.rados.0.smithi033.stdout:1775: expect (ObjNum 554 snap 149 seq_num 554) 2024-09-21T09:56:55.489 INFO:tasks.rados.rados.0.smithi033.stdout:1773: done (3 left) 2024-09-21T09:56:55.489 INFO:tasks.rados.rados.0.smithi033.stdout:1775: done (2 left) 2024-09-21T09:56:55.489 INFO:tasks.rados.rados.0.smithi033.stdout:1777: done (1 left) 2024-09-21T09:56:55.490 INFO:tasks.rados.rados.0.smithi033.stdout:1778: done (0 left) 2024-09-21T09:56:55.490 INFO:tasks.rados.rados.0.smithi033.stdout:1779: snap_create 2024-09-21T09:56:55.710 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:55 smithi033 ceph-mon[98546]: pgmap v210: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 162 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:56:55.710 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:55 smithi033 ceph-mon[98546]: osdmap e388: 8 total, 7 up, 8 in 2024-09-21T09:56:55.711 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:55 smithi033 ceph-mon[96568]: pgmap v210: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 162 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:56:55.711 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:55 smithi033 ceph-mon[96568]: osdmap e388: 8 total, 7 up, 8 in 2024-09-21T09:56:55.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:55 smithi105 ceph-mon[86649]: pgmap v210: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 162 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.1 MiB/s wr, 12 op/s; 18/204 objects degraded (8.824%) 2024-09-21T09:56:55.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:55 smithi105 ceph-mon[86649]: osdmap e388: 8 total, 7 up, 8 in 2024-09-21T09:56:56.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:56.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:56.508 INFO:tasks.rados.rados.0.smithi033.stdout:1779: done (0 left) 2024-09-21T09:56:56.508 INFO:tasks.rados.rados.0.smithi033.stdout:1780: read oid 37 snap -1 2024-09-21T09:56:56.508 INFO:tasks.rados.rados.0.smithi033.stdout:1780: expect (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:56:56.508 INFO:tasks.rados.rados.0.smithi033.stdout:1781: snap_remove snap 155 2024-09-21T09:56:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:56 smithi033 ceph-mon[98546]: osdmap e389: 8 total, 7 up, 8 in 2024-09-21T09:56:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:56 smithi033 ceph-mon[96568]: osdmap e389: 8 total, 7 up, 8 in 2024-09-21T09:56:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:56 smithi105 ceph-mon[86649]: osdmap e389: 8 total, 7 up, 8 in 2024-09-21T09:56:57.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:56:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:56:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:56:57.500 INFO:tasks.rados.rados.0.smithi033.stdout:1781: done (1 left) 2024-09-21T09:56:57.500 INFO:tasks.rados.rados.0.smithi033.stdout:1782: read oid 28 snap 157 2024-09-21T09:56:57.500 INFO:tasks.rados.rados.0.smithi033.stdout:1782: expect deleted 2024-09-21T09:56:57.500 INFO:tasks.rados.rados.0.smithi033.stdout:1783: snap_remove snap 154 2024-09-21T09:56:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:57 smithi105 ceph-mon[86649]: pgmap v213: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 748 KiB/s wr, 11 op/s; 22/231 objects degraded (9.524%) 2024-09-21T09:56:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:57 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 22/231 objects degraded (9.524%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:57 smithi105 ceph-mon[86649]: osdmap e390: 8 total, 7 up, 8 in 2024-09-21T09:56:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:57 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:57 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:57 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:57 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.859 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[98546]: pgmap v213: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 748 KiB/s wr, 11 op/s; 22/231 objects degraded (9.524%) 2024-09-21T09:56:57.859 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 22/231 objects degraded (9.524%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:57.859 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[98546]: osdmap e390: 8 total, 7 up, 8 in 2024-09-21T09:56:57.859 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.859 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.859 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.859 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.860 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[96568]: pgmap v213: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 748 KiB/s wr, 11 op/s; 22/231 objects degraded (9.524%) 2024-09-21T09:56:57.860 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 22/231 objects degraded (9.524%), 10 pgs degraded (PG_DEGRADED) 2024-09-21T09:56:57.860 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[96568]: osdmap e390: 8 total, 7 up, 8 in 2024-09-21T09:56:57.860 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.860 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.860 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:57.860 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:56:58.127 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:57 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[113392]: 2024-09-21T09:56:57.917+0000 7fe7b71af640 -1 osd.2 380 log_to_monitors true 2024-09-21T09:56:58.503 INFO:tasks.rados.rados.0.smithi033.stdout:1783: done (2 left) 2024-09-21T09:56:58.503 INFO:tasks.rados.rados.0.smithi033.stdout:1784: read oid 35 snap -1 2024-09-21T09:56:58.503 INFO:tasks.rados.rados.0.smithi033.stdout:1784: expect (ObjNum 562 snap 151 seq_num 562) 2024-09-21T09:56:58.503 INFO:tasks.rados.rados.0.smithi033.stdout:1785: copy_from oid 26 from oid 15 current snap is 158 2024-09-21T09:56:58.503 INFO:tasks.rados.rados.0.smithi033.stdout:1780: expect (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:56:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:58 smithi105 ceph-mon[86649]: osdmap e391: 8 total, 7 up, 8 in 2024-09-21T09:56:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:58 smithi105 ceph-mon[86649]: from='osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:56:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:58 smithi105 ceph-mon[86649]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:56:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:58 smithi105 ceph-mon[86649]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:56:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:58 smithi105 ceph-mon[86649]: osdmap e392: 8 total, 7 up, 8 in 2024-09-21T09:56:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:58 smithi105 ceph-mon[86649]: from='osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:58 smithi105 ceph-mon[86649]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:58.909 INFO:tasks.rados.rados.0.smithi033.stdout:1784: expect (ObjNum 562 snap 151 seq_num 562) 2024-09-21T09:56:58.931 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[113392]: 2024-09-21T09:56:58.543+0000 7fe7aef59640 -1 osd.2 380 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:56:58.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[98546]: osdmap e391: 8 total, 7 up, 8 in 2024-09-21T09:56:58.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[98546]: from='osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:56:58.931 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[98546]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[98546]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[98546]: osdmap e392: 8 total, 7 up, 8 in 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[98546]: from='osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[98546]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[96568]: osdmap e391: 8 total, 7 up, 8 in 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[96568]: from='osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[96568]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[96568]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[96568]: osdmap e392: 8 total, 7 up, 8 in 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[96568]: from='osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:58.932 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:58 smithi033 ceph-mon[96568]: from='osd.2 ' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:56:59.236 INFO:tasks.rados.rados.0.smithi033.stdout:1785: finishing copy_from racing read to smithi03385152-26 2024-09-21T09:56:59.236 INFO:tasks.rados.rados.0.smithi033.stdout:1785: finishing copy_from to smithi03385152-26 2024-09-21T09:56:59.236 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 670 (ObjNum 549 snap 148 seq_num 549) dirty exists 2024-09-21T09:56:59.236 INFO:tasks.rados.rados.0.smithi033.stdout:1780: done (3 left) 2024-09-21T09:56:59.236 INFO:tasks.rados.rados.0.smithi033.stdout:1782: done (2 left) 2024-09-21T09:56:59.236 INFO:tasks.rados.rados.0.smithi033.stdout:1784: done (1 left) 2024-09-21T09:56:59.236 INFO:tasks.rados.rados.0.smithi033.stdout:1785: done (0 left) 2024-09-21T09:56:59.236 INFO:tasks.rados.rados.0.smithi033.stdout:1786: read oid 13 snap -1 2024-09-21T09:56:59.237 INFO:tasks.rados.rados.0.smithi033.stdout:1786: expect deleted 2024-09-21T09:56:59.237 INFO:tasks.rados.rados.0.smithi033.stdout:1787: read oid 50 snap -1 2024-09-21T09:56:59.237 INFO:tasks.rados.rados.0.smithi033.stdout:1787: expect deleted 2024-09-21T09:56:59.237 INFO:tasks.rados.rados.0.smithi033.stdout:1788: snap_remove snap 153 2024-09-21T09:56:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1788: done (2 left) 2024-09-21T09:56:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1789: rmattr oid 41 current snap is 158 2024-09-21T09:56:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1789: done (2 left) 2024-09-21T09:56:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 158 2024-09-21T09:56:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:1790: seq_num 585 ranges {2039808=204800} 2024-09-21T09:56:59.537 INFO:tasks.rados.rados.0.smithi033.stdout:1790: writing smithi03385152-14 from 2039808 to 2244608 tid 1 2024-09-21T09:56:59.538 INFO:tasks.rados.rados.0.smithi033.stdout:1791: snap_remove snap 157 2024-09-21T09:56:59.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:56:59 smithi105 ceph-mon[86649]: pgmap v216: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 748 KiB/s wr, 11 op/s; 22/231 objects degraded (9.524%) 2024-09-21T09:56:59.921 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:56:59 smithi033 ceph-mon[98546]: pgmap v216: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 748 KiB/s wr, 11 op/s; 22/231 objects degraded (9.524%) 2024-09-21T09:56:59.922 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:56:59 smithi033 ceph-mon[96568]: pgmap v216: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 748 KiB/s wr, 11 op/s; 22/231 objects degraded (9.524%) 2024-09-21T09:57:00.530 INFO:tasks.rados.rados.0.smithi033.stdout:1786: done (3 left) 2024-09-21T09:57:00.530 INFO:tasks.rados.rados.0.smithi033.stdout:1787: done (2 left) 2024-09-21T09:57:00.530 INFO:tasks.rados.rados.0.smithi033.stdout:1791: done (1 left) 2024-09-21T09:57:00.530 INFO:tasks.rados.rados.0.smithi033.stdout:1792: delete oid 41 current snap is 158 2024-09-21T09:57:00.530 INFO:tasks.rados.rados.0.smithi033.stdout:1790: finishing write tid 1 to smithi03385152-14 2024-09-21T09:57:00.530 INFO:tasks.rados.rados.0.smithi033.stdout:1790: finishing write tid 2 to smithi03385152-14 2024-09-21T09:57:00.530 INFO:tasks.rados.rados.0.smithi033.stdout:1790: finishing write tid 3 to smithi03385152-14 2024-09-21T09:57:00.531 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 933 (ObjNum 585 snap 158 seq_num 585) dirty exists 2024-09-21T09:57:00.531 INFO:tasks.rados.rados.0.smithi033.stdout:1790: left oid 14 (ObjNum 585 snap 158 seq_num 585) 2024-09-21T09:57:00.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:00.552 INFO:tasks.rados.rados.0.smithi033.stdout:1790: done (1 left) 2024-09-21T09:57:00.552 INFO:tasks.rados.rados.0.smithi033.stdout:1792: done (0 left) 2024-09-21T09:57:00.552 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 158 2024-09-21T09:57:00.552 INFO:tasks.rados.rados.0.smithi033.stdout:1793: seq_num 587 ranges {3661824=237568} 2024-09-21T09:57:00.574 INFO:tasks.rados.rados.0.smithi033.stdout:1793: writing smithi03385152-6 from 3661824 to 3899392 tid 1 2024-09-21T09:57:00.574 INFO:tasks.rados.rados.0.smithi033.stdout:1794: snap_create 2024-09-21T09:57:00.577 INFO:tasks.rados.rados.0.smithi033.stdout:1793: finishing write tid 1 to smithi03385152-6 2024-09-21T09:57:00.577 INFO:tasks.rados.rados.0.smithi033.stdout:1793: finishing write tid 2 to smithi03385152-6 2024-09-21T09:57:00.578 INFO:tasks.rados.rados.0.smithi033.stdout:1793: finishing write tid 3 to smithi03385152-6 2024-09-21T09:57:00.578 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1033 (ObjNum 587 snap 158 seq_num 587) dirty exists 2024-09-21T09:57:00.579 INFO:tasks.rados.rados.0.smithi033.stdout:1793: left oid 6 (ObjNum 587 snap 158 seq_num 587) 2024-09-21T09:57:00.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-mon[96568]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:57:00.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-mon[96568]: osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654] boot 2024-09-21T09:57:00.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-mon[96568]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:57:00.622 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:57:00.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-mon[98546]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:57:00.622 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-mon[98546]: osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654] boot 2024-09-21T09:57:00.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-mon[98546]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:57:00.623 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:57:00.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:57:00 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:57:00.751+0000 7fc38e955640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T09:57:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:00 smithi105 ceph-mon[86649]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:57:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:00 smithi105 ceph-mon[86649]: osd.2 [v2:172.21.15.33:6818/586780654,v1:172.21.15.33:6819/586780654] boot 2024-09-21T09:57:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:00 smithi105 ceph-mon[86649]: osdmap e393: 8 total, 8 up, 8 in 2024-09-21T09:57:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-21T09:57:01.556 INFO:tasks.rados.rados.0.smithi033.stdout:1793: done (1 left) 2024-09-21T09:57:01.584 INFO:tasks.rados.rados.0.smithi033.stdout:1794: done (0 left) 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1795: rmattr oid 25 current snap is 159 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1796: copy_from oid 11 from oid 23 current snap is 159 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1797: rollback oid 39 current snap is 159 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 39 to 158 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1798: read oid 18 snap -1 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1798: expect (ObjNum 524 snap 142 seq_num 524) 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1799: read oid 35 snap -1 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1799: expect (ObjNum 562 snap 151 seq_num 562) 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1800: read oid 24 snap 158 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1800: expect deleted 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 159 2024-09-21T09:57:01.585 INFO:tasks.rados.rados.0.smithi033.stdout:1801: seq_num 588 ranges {9404416=253952} 2024-09-21T09:57:01.609 INFO:tasks.rados.rados.0.smithi033.stdout:1801: writing smithi03385152-22 from 9404416 to 9658368 tid 1 2024-09-21T09:57:01.610 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 159 2024-09-21T09:57:01.610 INFO:tasks.rados.rados.0.smithi033.stdout:1802: seq_num 589 ranges {1957888=270336} 2024-09-21T09:57:01.621 INFO:tasks.rados.rados.0.smithi033.stdout:1802: writing smithi03385152-49 from 1957888 to 2228224 tid 1 2024-09-21T09:57:01.622 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 159 2024-09-21T09:57:01.622 INFO:tasks.rados.rados.0.smithi033.stdout:1803: seq_num 590 ranges {9469952=286720} 2024-09-21T09:57:01.675 INFO:tasks.rados.rados.0.smithi033.stdout:1803: writing smithi03385152-2 from 9469952 to 9756672 tid 1 2024-09-21T09:57:01.675 INFO:tasks.rados.rados.0.smithi033.stdout:1804: read oid 47 snap -1 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:1804: expect (ObjNum 564 snap 151 seq_num 564) 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 159 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:1797: finishing rollback tid 0 to smithi03385152-39 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:1797: finishing rollback tid 1 to smithi03385152-39 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:1797: finishing rollback tid 2 to smithi03385152-39 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 237 (ObjNum 580 snap 156 seq_num 580) dirty exists 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 888 (ObjNum 535 snap 143 seq_num 535) dirty exists 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:1796: finishing copy_from racing read to smithi03385152-11 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:1796: finishing copy_from to smithi03385152-11 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 711 (ObjNum 581 snap 156 seq_num 581) dirty exists 2024-09-21T09:57:01.676 INFO:tasks.rados.rados.0.smithi033.stdout:1805: seq_num 591 ranges {6103040=303104} 2024-09-21T09:57:01.711 INFO:tasks.rados.rados.0.smithi033.stdout:1805: writing smithi03385152-33 from 6103040 to 6406144 tid 1 2024-09-21T09:57:01.711 INFO:tasks.rados.rados.0.smithi033.stdout:1795: done (10 left) 2024-09-21T09:57:01.711 INFO:tasks.rados.rados.0.smithi033.stdout:1796: done (9 left) 2024-09-21T09:57:01.711 INFO:tasks.rados.rados.0.smithi033.stdout:1797: done (8 left) 2024-09-21T09:57:01.711 INFO:tasks.rados.rados.0.smithi033.stdout:1800: done (7 left) 2024-09-21T09:57:01.711 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 159 2024-09-21T09:57:01.711 INFO:tasks.rados.rados.0.smithi033.stdout:1798: expect (ObjNum 524 snap 142 seq_num 524) 2024-09-21T09:57:01.857 INFO:tasks.rados.rados.0.smithi033.stdout:1799: expect (ObjNum 562 snap 151 seq_num 562) 2024-09-21T09:57:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:57:01] "GET /metrics HTTP/1.1" 200 36178 "" "Prometheus/2.51.0" 2024-09-21T09:57:01.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: pgmap v219: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 0 B/s wr, 1 op/s; 22/234 objects degraded (9.402%) 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:57:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: pgmap v219: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 0 B/s wr, 1 op/s; 22/234 objects degraded (9.402%) 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:01.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:01.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: pgmap v219: 121 pgs: 27 active+undersized, 10 active+undersized+degraded, 84 active+clean; 165 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 0 B/s wr, 1 op/s; 22/234 objects degraded (9.402%) 2024-09-21T09:57:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: osdmap e394: 8 total, 8 up, 8 in 2024-09-21T09:57:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:02.179 INFO:tasks.rados.rados.0.smithi033.stdout:1801: finishing write tid 1 to smithi03385152-22 2024-09-21T09:57:02.179 INFO:tasks.rados.rados.0.smithi033.stdout:1801: finishing write tid 2 to smithi03385152-22 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1801: finishing write tid 3 to smithi03385152-22 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 938 (ObjNum 588 snap 159 seq_num 588) dirty exists 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1801: left oid 22 (ObjNum 588 snap 159 seq_num 588) 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1802: finishing write tid 1 to smithi03385152-49 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1802: finishing write tid 2 to smithi03385152-49 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1802: finishing write tid 3 to smithi03385152-49 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1081 (ObjNum 589 snap 159 seq_num 589) dirty exists 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1802: left oid 49 (ObjNum 589 snap 159 seq_num 589) 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1803: finishing write tid 1 to smithi03385152-2 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1803: finishing write tid 2 to smithi03385152-2 2024-09-21T09:57:02.180 INFO:tasks.rados.rados.0.smithi033.stdout:1803: finishing write tid 3 to smithi03385152-2 2024-09-21T09:57:02.181 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 890 (ObjNum 590 snap 159 seq_num 590) dirty exists 2024-09-21T09:57:02.181 INFO:tasks.rados.rados.0.smithi033.stdout:1803: left oid 2 (ObjNum 590 snap 159 seq_num 590) 2024-09-21T09:57:02.181 INFO:tasks.rados.rados.0.smithi033.stdout:1806: seq_num 592 ranges {1957888=319488} 2024-09-21T09:57:02.192 INFO:tasks.rados.rados.0.smithi033.stdout:1806: writing smithi03385152-36 from 1957888 to 2277376 tid 1 2024-09-21T09:57:02.192 INFO:tasks.rados.rados.0.smithi033.stdout:1798: done (7 left) 2024-09-21T09:57:02.192 INFO:tasks.rados.rados.0.smithi033.stdout:1799: done (6 left) 2024-09-21T09:57:02.192 INFO:tasks.rados.rados.0.smithi033.stdout:1801: done (5 left) 2024-09-21T09:57:02.192 INFO:tasks.rados.rados.0.smithi033.stdout:1802: done (4 left) 2024-09-21T09:57:02.192 INFO:tasks.rados.rados.0.smithi033.stdout:1803: done (3 left) 2024-09-21T09:57:02.192 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 159 2024-09-21T09:57:02.192 INFO:tasks.rados.rados.0.smithi033.stdout:1804: expect (ObjNum 564 snap 151 seq_num 564) 2024-09-21T09:57:02.752 INFO:tasks.rados.rados.0.smithi033.stdout:1805: finishing write tid 1 to smithi03385152-33 2024-09-21T09:57:02.753 INFO:tasks.rados.rados.0.smithi033.stdout:1805: finishing write tid 2 to smithi03385152-33 2024-09-21T09:57:02.753 INFO:tasks.rados.rados.0.smithi033.stdout:1805: finishing write tid 3 to smithi03385152-33 2024-09-21T09:57:02.753 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1035 (ObjNum 591 snap 159 seq_num 591) dirty exists 2024-09-21T09:57:02.753 INFO:tasks.rados.rados.0.smithi033.stdout:1805: left oid 33 (ObjNum 591 snap 159 seq_num 591) 2024-09-21T09:57:02.753 INFO:tasks.rados.rados.0.smithi033.stdout:1807: seq_num 593 ranges {2113536=335872} 2024-09-21T09:57:02.766 INFO:tasks.rados.rados.0.smithi033.stdout:1807: writing smithi03385152-23 from 2113536 to 2449408 tid 1 2024-09-21T09:57:02.766 INFO:tasks.rados.rados.0.smithi033.stdout:1804: done (3 left) 2024-09-21T09:57:02.766 INFO:tasks.rados.rados.0.smithi033.stdout:1805: done (2 left) 2024-09-21T09:57:02.766 INFO:tasks.rados.rados.0.smithi033.stdout:1808: snap_create 2024-09-21T09:57:02.766 INFO:tasks.rados.rados.0.smithi033.stdout:1806: finishing write tid 1 to smithi03385152-36 2024-09-21T09:57:02.766 INFO:tasks.rados.rados.0.smithi033.stdout:1806: finishing write tid 2 to smithi03385152-36 2024-09-21T09:57:02.767 INFO:tasks.rados.rados.0.smithi033.stdout:1806: finishing write tid 3 to smithi03385152-36 2024-09-21T09:57:02.767 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1140 (ObjNum 592 snap 159 seq_num 592) dirty exists 2024-09-21T09:57:02.767 INFO:tasks.rados.rados.0.smithi033.stdout:1806: left oid 36 (ObjNum 592 snap 159 seq_num 592) 2024-09-21T09:57:02.768 INFO:tasks.rados.rados.0.smithi033.stdout:1807: finishing write tid 1 to smithi03385152-23 2024-09-21T09:57:02.769 INFO:tasks.rados.rados.0.smithi033.stdout:1807: finishing write tid 2 to smithi03385152-23 2024-09-21T09:57:02.769 INFO:tasks.rados.rados.0.smithi033.stdout:1807: finishing write tid 3 to smithi03385152-23 2024-09-21T09:57:02.769 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 713 (ObjNum 593 snap 159 seq_num 593) dirty exists 2024-09-21T09:57:02.769 INFO:tasks.rados.rados.0.smithi033.stdout:1807: left oid 23 (ObjNum 593 snap 159 seq_num 593) 2024-09-21T09:57:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:02 smithi033 ceph-mon[98546]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:57:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:02 smithi033 ceph-mon[96568]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:57:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:02 smithi105 ceph-mon[86649]: osdmap e395: 8 total, 8 up, 8 in 2024-09-21T09:57:03.568 INFO:tasks.rados.rados.0.smithi033.stdout:1806: done (2 left) 2024-09-21T09:57:03.570 INFO:tasks.rados.rados.0.smithi033.stdout:1807: done (1 left) 2024-09-21T09:57:03.571 INFO:tasks.rados.rados.0.smithi033.stdout:1808: done (0 left) 2024-09-21T09:57:03.571 INFO:tasks.rados.rados.0.smithi033.stdout:1809: delete oid 50 current snap is 160 2024-09-21T09:57:03.572 INFO:tasks.rados.rados.0.smithi033.stdout:1809: done (0 left) 2024-09-21T09:57:03.572 INFO:tasks.rados.rados.0.smithi033.stdout:1810: rmattr oid 47 current snap is 160 2024-09-21T09:57:03.572 INFO:tasks.rados.rados.0.smithi033.stdout:1811: delete oid 15 current snap is 160 2024-09-21T09:57:03.575 INFO:tasks.rados.rados.0.smithi033.stdout:1811: done (1 left) 2024-09-21T09:57:03.575 INFO:tasks.rados.rados.0.smithi033.stdout:1812: snap_remove snap 156 2024-09-21T09:57:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:03 smithi033 ceph-mon[98546]: pgmap v222: 121 pgs: 10 active+undersized, 7 active+undersized+degraded, 104 active+clean; 152 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 2.2 MiB/s wr, 4 op/s; 17/213 objects degraded (7.981%) 2024-09-21T09:57:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:03 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 17/213 objects degraded (7.981%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:03 smithi033 ceph-mon[96568]: pgmap v222: 121 pgs: 10 active+undersized, 7 active+undersized+degraded, 104 active+clean; 152 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 2.2 MiB/s wr, 4 op/s; 17/213 objects degraded (7.981%) 2024-09-21T09:57:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:03 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 17/213 objects degraded (7.981%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:03 smithi105 ceph-mon[86649]: pgmap v222: 121 pgs: 10 active+undersized, 7 active+undersized+degraded, 104 active+clean; 152 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 9.8 MiB/s rd, 2.2 MiB/s wr, 4 op/s; 17/213 objects degraded (7.981%) 2024-09-21T09:57:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:03 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 17/213 objects degraded (7.981%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:04.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:04.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 656 (ObjNum 564 snap 151 seq_num 564) dirty exists 2024-09-21T09:57:04.576 INFO:tasks.rados.rados.0.smithi033.stdout:1810: done (1 left) 2024-09-21T09:57:04.576 INFO:tasks.rados.rados.0.smithi033.stdout:1812: done (0 left) 2024-09-21T09:57:04.576 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 160 2024-09-21T09:57:04.576 INFO:tasks.rados.rados.0.smithi033.stdout:1813: seq_num 596 ranges {2449408=385024} 2024-09-21T09:57:04.592 INFO:tasks.rados.rados.0.smithi033.stdout:1813: writing smithi03385152-23 from 2449408 to 2834432 tid 1 2024-09-21T09:57:04.592 INFO:tasks.rados.rados.0.smithi033.stdout:1814: read oid 15 snap -1 2024-09-21T09:57:04.592 INFO:tasks.rados.rados.0.smithi033.stdout:1814: expect deleted 2024-09-21T09:57:04.592 INFO:tasks.rados.rados.0.smithi033.stdout:1815: setattr oid 36 current snap is 160 2024-09-21T09:57:04.593 INFO:tasks.rados.rados.0.smithi033.stdout:1814: done (2 left) 2024-09-21T09:57:04.593 INFO:tasks.rados.rados.0.smithi033.stdout:1816: snap_create 2024-09-21T09:57:04.595 INFO:tasks.rados.rados.0.smithi033.stdout:1813: finishing write tid 1 to smithi03385152-23 2024-09-21T09:57:04.595 INFO:tasks.rados.rados.0.smithi033.stdout:1813: finishing write tid 2 to smithi03385152-23 2024-09-21T09:57:04.596 INFO:tasks.rados.rados.0.smithi033.stdout:1813: finishing write tid 3 to smithi03385152-23 2024-09-21T09:57:04.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 715 (ObjNum 596 snap 160 seq_num 596) dirty exists 2024-09-21T09:57:04.596 INFO:tasks.rados.rados.0.smithi033.stdout:1813: left oid 23 (ObjNum 596 snap 160 seq_num 596) 2024-09-21T09:57:04.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1144 (ObjNum 592 snap 159 seq_num 592) dirty exists 2024-09-21T09:57:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:04 smithi033 ceph-mon[98546]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:57:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:04 smithi033 ceph-mon[96568]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:57:05.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:04 smithi105 ceph-mon[86649]: osdmap e396: 8 total, 8 up, 8 in 2024-09-21T09:57:05.579 INFO:tasks.rados.rados.0.smithi033.stdout:1813: done (2 left) 2024-09-21T09:57:05.579 INFO:tasks.rados.rados.0.smithi033.stdout:1815: done (1 left) 2024-09-21T09:57:05.579 INFO:tasks.rados.rados.0.smithi033.stdout:1816: done (0 left) 2024-09-21T09:57:05.580 INFO:tasks.rados.rados.0.smithi033.stdout:1817: delete oid 2 current snap is 161 2024-09-21T09:57:05.588 INFO:tasks.rados.rados.0.smithi033.stdout:1817: done (0 left) 2024-09-21T09:57:05.588 INFO:tasks.rados.rados.0.smithi033.stdout:1818: copy_from oid 29 from oid 30 current snap is 161 2024-09-21T09:57:05.588 INFO:tasks.rados.rados.0.smithi033.stdout:1819: read oid 1 snap 159 2024-09-21T09:57:05.588 INFO:tasks.rados.rados.0.smithi033.stdout:1819: expect (ObjNum 404 snap 110 seq_num 404) 2024-09-21T09:57:05.588 INFO:tasks.rados.rados.0.smithi033.stdout:1820: rollback oid 17 current snap is 161 2024-09-21T09:57:05.589 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 160 2024-09-21T09:57:05.589 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 161 2024-09-21T09:57:05.589 INFO:tasks.rados.rados.0.smithi033.stdout:1821: seq_num 598 ranges {2228224=417792} 2024-09-21T09:57:05.603 INFO:tasks.rados.rados.0.smithi033.stdout:1821: writing smithi03385152-49 from 2228224 to 2646016 tid 1 2024-09-21T09:57:05.603 INFO:tasks.rados.rados.0.smithi033.stdout:1822: setattr oid 5 current snap is 161 2024-09-21T09:57:05.603 INFO:tasks.rados.rados.0.smithi033.stdout:1820: finishing rollback tid 0 to smithi03385152-17 2024-09-21T09:57:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:1820: finishing rollback tid 1 to smithi03385152-17 2024-09-21T09:57:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:1820: finishing rollback tid 2 to smithi03385152-17 2024-09-21T09:57:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 1084 (ObjNum 541 snap 145 seq_num 541) dirty exists 2024-09-21T09:57:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:1819: expect (ObjNum 404 snap 110 seq_num 404) 2024-09-21T09:57:05.657 INFO:tasks.rados.rados.0.smithi033.stdout:1821: finishing write tid 1 to smithi03385152-49 2024-09-21T09:57:05.657 INFO:tasks.rados.rados.0.smithi033.stdout:1821: finishing write tid 2 to smithi03385152-49 2024-09-21T09:57:05.657 INFO:tasks.rados.rados.0.smithi033.stdout:1821: finishing write tid 3 to smithi03385152-49 2024-09-21T09:57:05.657 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1086 (ObjNum 598 snap 161 seq_num 598) dirty exists 2024-09-21T09:57:05.657 INFO:tasks.rados.rados.0.smithi033.stdout:1821: left oid 49 (ObjNum 598 snap 161 seq_num 598) 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1818: finishing copy_from racing read to smithi03385152-29 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1818: finishing copy_from to smithi03385152-29 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 885 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1818: done (4 left) 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1819: done (3 left) 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1820: done (2 left) 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1821: done (1 left) 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1823: read oid 41 snap 160 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1823: expect deleted 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1824: rmattr oid 9 current snap is 161 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1824: done (2 left) 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 161 2024-09-21T09:57:05.658 INFO:tasks.rados.rados.0.smithi033.stdout:1825: seq_num 599 ranges {7503872=434176} 2024-09-21T09:57:05.700 INFO:tasks.rados.rados.0.smithi033.stdout:1825: writing smithi03385152-16 from 7503872 to 7938048 tid 1 2024-09-21T09:57:05.700 INFO:tasks.rados.rados.0.smithi033.stdout:1826: rollback oid 24 current snap is 161 2024-09-21T09:57:05.700 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 160 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 241 (ObjNum 526 snap 143 seq_num 526) dirty exists 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:1822: done (3 left) 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:1823: done (2 left) 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:1827: copy_from oid 35 from oid 32 current snap is 161 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:1828: rollback oid 4 current snap is 161 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 4 to 160 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:1829: setattr oid 8 current snap is 161 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:1826: finishing rollback tid 1 to smithi03385152-24 2024-09-21T09:57:05.701 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 0 (ObjNum 544235886 snap 171258673 seq_num 304) dirty dne 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1826: done (4 left) 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1830: read oid 41 snap -1 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1830: expect deleted 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1831: copy_from oid 21 from oid 29 current snap is 161 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1832: copy_from oid 1 from oid 25 current snap is 161 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1833: read oid 39 snap 160 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1833: expect (ObjNum 580 snap 156 seq_num 580) 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1834: read oid 43 snap 158 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1834: expect deleted 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1835: copy_from oid 6 from oid 11 current snap is 161 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 161 2024-09-21T09:57:05.703 INFO:tasks.rados.rados.0.smithi033.stdout:1836: seq_num 600 ranges {0=450560} 2024-09-21T09:57:05.706 INFO:tasks.rados.rados.0.smithi033.stdout:1836: writing smithi03385152-19 from 0 to 450560 tid 1 2024-09-21T09:57:05.706 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 161 2024-09-21T09:57:05.706 INFO:tasks.rados.rados.0.smithi033.stdout:1837: seq_num 601 ranges {9404416=475136} 2024-09-21T09:57:05.762 INFO:tasks.rados.rados.0.smithi033.stdout:1837: writing smithi03385152-30 from 9404416 to 9879552 tid 1 2024-09-21T09:57:05.762 INFO:tasks.rados.rados.0.smithi033.stdout:1838: copy_from oid 47 from oid 49 current snap is 161 2024-09-21T09:57:05.762 INFO:tasks.rados.rados.0.smithi033.stdout:1828: finishing rollback tid 0 to smithi03385152-4 2024-09-21T09:57:05.762 INFO:tasks.rados.rados.0.smithi033.stdout:1828: finishing rollback tid 1 to smithi03385152-4 2024-09-21T09:57:05.763 INFO:tasks.rados.rados.0.smithi033.stdout:1830: done (12 left) 2024-09-21T09:57:05.763 INFO:tasks.rados.rados.0.smithi033.stdout:1834: done (11 left) 2024-09-21T09:57:05.763 INFO:tasks.rados.rados.0.smithi033.stdout:1839: delete oid 12 current snap is 161 2024-09-21T09:57:05.763 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 678 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-21T09:57:05.763 INFO:tasks.rados.rados.0.smithi033.stdout:1833: expect (ObjNum 580 snap 156 seq_num 580) 2024-09-21T09:57:05.765 INFO:tasks.rados.rados.0.smithi033.stdout:1828: finishing rollback tid 2 to smithi03385152-4 2024-09-21T09:57:05.765 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1015 (ObjNum 572 snap 154 seq_num 572) dirty exists 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:1836: finishing write tid 1 to smithi03385152-19 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:1825: finishing write tid 1 to smithi03385152-16 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:1825: finishing write tid 2 to smithi03385152-16 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:1836: finishing write tid 2 to smithi03385152-19 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:1836: finishing write tid 3 to smithi03385152-19 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1002 (ObjNum 600 snap 161 seq_num 600) dirty exists 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:1836: left oid 19 (ObjNum 600 snap 161 seq_num 600) 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:1827: finishing copy_from racing read to smithi03385152-35 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:1825: finishing write tid 3 to smithi03385152-16 2024-09-21T09:57:05.766 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 1148 (ObjNum 599 snap 161 seq_num 599) dirty exists 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1825: left oid 16 (ObjNum 599 snap 161 seq_num 599) 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1835: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1827: finishing copy_from to smithi03385152-35 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 169 (ObjNum 574 snap 155 seq_num 574) dirty exists 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1832: finishing copy_from racing read to smithi03385152-1 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1832: finishing copy_from to smithi03385152-1 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 888 (ObjNum 535 snap 143 seq_num 535) dirty exists 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1835: finishing copy_from to smithi03385152-6 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 711 (ObjNum 581 snap 156 seq_num 581) dirty exists 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1831: finishing copy_from racing read to smithi03385152-21 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1831: finishing copy_from to smithi03385152-21 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 885 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1837: finishing write tid 1 to smithi03385152-30 2024-09-21T09:57:05.767 INFO:tasks.rados.rados.0.smithi033.stdout:1837: finishing write tid 2 to smithi03385152-30 2024-09-21T09:57:05.768 INFO:tasks.rados.rados.0.smithi033.stdout:1825: done (11 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1827: done (10 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1828: done (9 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1829: done (8 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1831: done (7 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1832: done (6 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1833: done (5 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1835: done (4 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1836: done (3 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1839: done (2 left) 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1840: read oid 9 snap 160 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1840: expect deleted 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1841: delete oid 20 current snap is 161 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:1837: finishing write tid 3 to smithi03385152-30 2024-09-21T09:57:05.769 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 1153 (ObjNum 601 snap 161 seq_num 601) dirty exists 2024-09-21T09:57:05.770 INFO:tasks.rados.rados.0.smithi033.stdout:1837: left oid 30 (ObjNum 601 snap 161 seq_num 601) 2024-09-21T09:57:05.770 INFO:tasks.rados.rados.0.smithi033.stdout:1837: done (3 left) 2024-09-21T09:57:05.770 INFO:tasks.rados.rados.0.smithi033.stdout:1840: done (2 left) 2024-09-21T09:57:05.770 INFO:tasks.rados.rados.0.smithi033.stdout:1841: done (1 left) 2024-09-21T09:57:05.770 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 161 2024-09-21T09:57:05.770 INFO:tasks.rados.rados.0.smithi033.stdout:1842: seq_num 604 ranges {114688=524288} 2024-09-21T09:57:05.772 INFO:tasks.rados.rados.0.smithi033.stdout:1842: writing smithi03385152-39 from 114688 to 638976 tid 1 2024-09-21T09:57:05.772 INFO:tasks.rados.rados.0.smithi033.stdout:1843: snap_create 2024-09-21T09:57:05.775 INFO:tasks.rados.rados.0.smithi033.stdout:1838: finishing copy_from racing read to smithi03385152-47 2024-09-21T09:57:05.775 INFO:tasks.rados.rados.0.smithi033.stdout:1842: finishing write tid 1 to smithi03385152-39 2024-09-21T09:57:05.776 INFO:tasks.rados.rados.0.smithi033.stdout:1842: finishing write tid 2 to smithi03385152-39 2024-09-21T09:57:05.776 INFO:tasks.rados.rados.0.smithi033.stdout:1838: finishing copy_from to smithi03385152-47 2024-09-21T09:57:05.776 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1086 (ObjNum 598 snap 161 seq_num 598) dirty exists 2024-09-21T09:57:05.777 INFO:tasks.rados.rados.0.smithi033.stdout:1842: finishing write tid 3 to smithi03385152-39 2024-09-21T09:57:05.777 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 244 (ObjNum 604 snap 161 seq_num 604) dirty exists 2024-09-21T09:57:05.777 INFO:tasks.rados.rados.0.smithi033.stdout:1842: left oid 39 (ObjNum 604 snap 161 seq_num 604) 2024-09-21T09:57:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:05 smithi033 ceph-mon[98546]: pgmap v224: 121 pgs: 4 active+undersized, 117 active+clean; 148 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.8 MiB/s wr, 4 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-09-21T09:57:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:05 smithi033 ceph-mon[98546]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/213 objects degraded (7.981%), 7 pgs degraded) 2024-09-21T09:57:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:05 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T09:57:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:05 smithi033 ceph-mon[98546]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:57:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:05 smithi033 ceph-mon[96568]: pgmap v224: 121 pgs: 4 active+undersized, 117 active+clean; 148 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.8 MiB/s wr, 4 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-09-21T09:57:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:05 smithi033 ceph-mon[96568]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/213 objects degraded (7.981%), 7 pgs degraded) 2024-09-21T09:57:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:05 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T09:57:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:05 smithi033 ceph-mon[96568]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:57:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:05 smithi105 ceph-mon[86649]: pgmap v224: 121 pgs: 4 active+undersized, 117 active+clean; 148 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.8 MiB/s wr, 4 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-09-21T09:57:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:05 smithi105 ceph-mon[86649]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 17/213 objects degraded (7.981%), 7 pgs degraded) 2024-09-21T09:57:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:05 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T09:57:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:05 smithi105 ceph-mon[86649]: osdmap e397: 8 total, 8 up, 8 in 2024-09-21T09:57:06.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:05.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:06.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:05.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:06.599 INFO:tasks.rados.rados.0.smithi033.stdout:1838: done (2 left) 2024-09-21T09:57:06.599 INFO:tasks.rados.rados.0.smithi033.stdout:1842: done (1 left) 2024-09-21T09:57:06.599 INFO:tasks.rados.rados.0.smithi033.stdout:1843: done (0 left) 2024-09-21T09:57:06.599 INFO:tasks.rados.rados.0.smithi033.stdout:1844: read oid 14 snap 161 2024-09-21T09:57:06.599 INFO:tasks.rados.rados.0.smithi033.stdout:1844: expect (ObjNum 585 snap 158 seq_num 585) 2024-09-21T09:57:06.599 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 162 2024-09-21T09:57:06.599 INFO:tasks.rados.rados.0.smithi033.stdout:1845: seq_num 605 ranges {638976=540672} 2024-09-21T09:57:06.605 INFO:tasks.rados.rados.0.smithi033.stdout:1845: writing smithi03385152-39 from 638976 to 1179648 tid 1 2024-09-21T09:57:06.605 INFO:tasks.rados.rados.0.smithi033.stdout:1846: delete oid 1 current snap is 162 2024-09-21T09:57:06.608 INFO:tasks.rados.rados.0.smithi033.stdout:1846: done (2 left) 2024-09-21T09:57:06.608 INFO:tasks.rados.rados.0.smithi033.stdout:1847: rollback oid 36 current snap is 162 2024-09-21T09:57:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 160 2024-09-21T09:57:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:1848: read oid 3 snap 159 2024-09-21T09:57:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:1848: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:57:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 162 2024-09-21T09:57:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:1849: seq_num 607 ranges {9658368=573440} 2024-09-21T09:57:06.665 INFO:tasks.rados.rados.0.smithi033.stdout:1849: writing smithi03385152-22 from 9658368 to 10231808 tid 1 2024-09-21T09:57:06.665 INFO:tasks.rados.rados.0.smithi033.stdout:1850: snap_remove snap 158 2024-09-21T09:57:06.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:06 smithi105 ceph-mon[86649]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:57:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:06 smithi033 ceph-mon[98546]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:57:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:06 smithi033 ceph-mon[96568]: osdmap e398: 8 total, 8 up, 8 in 2024-09-21T09:57:07.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:06.948Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:07.600 INFO:tasks.rados.rados.0.smithi033.stdout:1850: done (5 left) 2024-09-21T09:57:07.600 INFO:tasks.rados.rados.0.smithi033.stdout:1851: read oid 18 snap -1 2024-09-21T09:57:07.600 INFO:tasks.rados.rados.0.smithi033.stdout:1851: expect (ObjNum 524 snap 142 seq_num 524) 2024-09-21T09:57:07.601 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 162 2024-09-21T09:57:07.601 INFO:tasks.rados.rados.0.smithi033.stdout:1844: expect (ObjNum 585 snap 158 seq_num 585) 2024-09-21T09:57:07.702 INFO:tasks.rados.rados.0.smithi033.stdout:1845: finishing write tid 1 to smithi03385152-39 2024-09-21T09:57:07.702 INFO:tasks.rados.rados.0.smithi033.stdout:1848: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:57:07.712 INFO:tasks.rados.rados.0.smithi033.stdout:1847: finishing rollback tid 0 to smithi03385152-36 2024-09-21T09:57:07.712 INFO:tasks.rados.rados.0.smithi033.stdout:1845: finishing write tid 2 to smithi03385152-39 2024-09-21T09:57:07.712 INFO:tasks.rados.rados.0.smithi033.stdout:1847: finishing rollback tid 1 to smithi03385152-36 2024-09-21T09:57:07.712 INFO:tasks.rados.rados.0.smithi033.stdout:1847: finishing rollback tid 2 to smithi03385152-36 2024-09-21T09:57:07.712 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1157 (ObjNum 592 snap 159 seq_num 592) dirty exists 2024-09-21T09:57:07.712 INFO:tasks.rados.rados.0.smithi033.stdout:1845: finishing write tid 3 to smithi03385152-39 2024-09-21T09:57:07.712 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 247 (ObjNum 605 snap 162 seq_num 605) dirty exists 2024-09-21T09:57:07.712 INFO:tasks.rados.rados.0.smithi033.stdout:1845: left oid 39 (ObjNum 605 snap 162 seq_num 605) 2024-09-21T09:57:07.713 INFO:tasks.rados.rados.0.smithi033.stdout:1849: finishing write tid 1 to smithi03385152-22 2024-09-21T09:57:07.713 INFO:tasks.rados.rados.0.smithi033.stdout:1849: finishing write tid 2 to smithi03385152-22 2024-09-21T09:57:07.713 INFO:tasks.rados.rados.0.smithi033.stdout:1852: seq_num 608 ranges {5857280=589824} 2024-09-21T09:57:07.747 INFO:tasks.rados.rados.0.smithi033.stdout:1852: writing smithi03385152-44 from 5857280 to 6447104 tid 1 2024-09-21T09:57:07.747 INFO:tasks.rados.rados.0.smithi033.stdout:1844: done (6 left) 2024-09-21T09:57:07.748 INFO:tasks.rados.rados.0.smithi033.stdout:1845: done (5 left) 2024-09-21T09:57:07.748 INFO:tasks.rados.rados.0.smithi033.stdout:1847: done (4 left) 2024-09-21T09:57:07.748 INFO:tasks.rados.rados.0.smithi033.stdout:1848: done (3 left) 2024-09-21T09:57:07.748 INFO:tasks.rados.rados.0.smithi033.stdout:1853: read oid 31 snap -1 2024-09-21T09:57:07.748 INFO:tasks.rados.rados.0.smithi033.stdout:1853: expect (ObjNum 548 snap 148 seq_num 548) 2024-09-21T09:57:07.748 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 162 2024-09-21T09:57:07.748 INFO:tasks.rados.rados.0.smithi033.stdout:1854: seq_num 609 ranges {2244608=606208} 2024-09-21T09:57:07.763 INFO:tasks.rados.rados.0.smithi033.stdout:1854: writing smithi03385152-14 from 2244608 to 2850816 tid 1 2024-09-21T09:57:07.763 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 162 2024-09-21T09:57:07.763 INFO:tasks.rados.rados.0.smithi033.stdout:1849: finishing write tid 3 to smithi03385152-22 2024-09-21T09:57:07.763 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 945 (ObjNum 607 snap 162 seq_num 607) dirty exists 2024-09-21T09:57:07.763 INFO:tasks.rados.rados.0.smithi033.stdout:1849: left oid 22 (ObjNum 607 snap 162 seq_num 607) 2024-09-21T09:57:07.763 INFO:tasks.rados.rados.0.smithi033.stdout:1851: expect (ObjNum 524 snap 142 seq_num 524) 2024-09-21T09:57:07.907 INFO:tasks.rados.rados.0.smithi033.stdout:1852: finishing write tid 1 to smithi03385152-44 2024-09-21T09:57:07.908 INFO:tasks.rados.rados.0.smithi033.stdout:1852: finishing write tid 2 to smithi03385152-44 2024-09-21T09:57:07.908 INFO:tasks.rados.rados.0.smithi033.stdout:1852: finishing write tid 3 to smithi03385152-44 2024-09-21T09:57:07.908 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1017 (ObjNum 608 snap 162 seq_num 608) dirty exists 2024-09-21T09:57:07.908 INFO:tasks.rados.rados.0.smithi033.stdout:1852: left oid 44 (ObjNum 608 snap 162 seq_num 608) 2024-09-21T09:57:07.908 INFO:tasks.rados.rados.0.smithi033.stdout:1853: expect (ObjNum 548 snap 148 seq_num 548) 2024-09-21T09:57:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:07 smithi033 ceph-mon[98546]: pgmap v227: 121 pgs: 121 active+clean; 163 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 22 op/s; 2.4 MiB/s, 1 objects/s recovering 2024-09-21T09:57:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:07 smithi033 ceph-mon[98546]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:57:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:07 smithi033 ceph-mon[96568]: pgmap v227: 121 pgs: 121 active+clean; 163 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 22 op/s; 2.4 MiB/s, 1 objects/s recovering 2024-09-21T09:57:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:07 smithi033 ceph-mon[96568]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:57:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:07 smithi105 ceph-mon[86649]: pgmap v227: 121 pgs: 121 active+clean; 163 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.3 MiB/s wr, 22 op/s; 2.4 MiB/s, 1 objects/s recovering 2024-09-21T09:57:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:07 smithi105 ceph-mon[86649]: osdmap e399: 8 total, 8 up, 8 in 2024-09-21T09:57:08.110 INFO:tasks.rados.rados.0.smithi033.stdout:1854: finishing write tid 1 to smithi03385152-14 2024-09-21T09:57:08.110 INFO:tasks.rados.rados.0.smithi033.stdout:1854: finishing write tid 2 to smithi03385152-14 2024-09-21T09:57:08.110 INFO:tasks.rados.rados.0.smithi033.stdout:1854: finishing write tid 3 to smithi03385152-14 2024-09-21T09:57:08.111 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 950 (ObjNum 609 snap 162 seq_num 609) dirty exists 2024-09-21T09:57:08.111 INFO:tasks.rados.rados.0.smithi033.stdout:1854: left oid 14 (ObjNum 609 snap 162 seq_num 609) 2024-09-21T09:57:08.111 INFO:tasks.rados.rados.0.smithi033.stdout:1855: seq_num 610 ranges {2408448=622592} 2024-09-21T09:57:08.127 INFO:tasks.rados.rados.0.smithi033.stdout:1855: writing smithi03385152-45 from 2408448 to 3031040 tid 1 2024-09-21T09:57:08.127 INFO:tasks.rados.rados.0.smithi033.stdout:1849: done (5 left) 2024-09-21T09:57:08.127 INFO:tasks.rados.rados.0.smithi033.stdout:1851: done (4 left) 2024-09-21T09:57:08.127 INFO:tasks.rados.rados.0.smithi033.stdout:1852: done (3 left) 2024-09-21T09:57:08.127 INFO:tasks.rados.rados.0.smithi033.stdout:1853: done (2 left) 2024-09-21T09:57:08.127 INFO:tasks.rados.rados.0.smithi033.stdout:1854: done (1 left) 2024-09-21T09:57:08.127 INFO:tasks.rados.rados.0.smithi033.stdout:1856: delete oid 1 current snap is 162 2024-09-21T09:57:08.129 INFO:tasks.rados.rados.0.smithi033.stdout:1856: done (1 left) 2024-09-21T09:57:08.129 INFO:tasks.rados.rados.0.smithi033.stdout:1857: read oid 39 snap -1 2024-09-21T09:57:08.129 INFO:tasks.rados.rados.0.smithi033.stdout:1857: expect (ObjNum 605 snap 162 seq_num 605) 2024-09-21T09:57:08.129 INFO:tasks.rados.rados.0.smithi033.stdout:1858: read oid 31 snap -1 2024-09-21T09:57:08.129 INFO:tasks.rados.rados.0.smithi033.stdout:1858: expect (ObjNum 548 snap 148 seq_num 548) 2024-09-21T09:57:08.129 INFO:tasks.rados.rados.0.smithi033.stdout:1859: snap_create 2024-09-21T09:57:08.131 INFO:tasks.rados.rados.0.smithi033.stdout:1855: finishing write tid 1 to smithi03385152-45 2024-09-21T09:57:08.132 INFO:tasks.rados.rados.0.smithi033.stdout:1855: finishing write tid 2 to smithi03385152-45 2024-09-21T09:57:08.134 INFO:tasks.rados.rados.0.smithi033.stdout:1855: finishing write tid 3 to smithi03385152-45 2024-09-21T09:57:08.134 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 953 (ObjNum 610 snap 162 seq_num 610) dirty exists 2024-09-21T09:57:08.134 INFO:tasks.rados.rados.0.smithi033.stdout:1855: left oid 45 (ObjNum 610 snap 162 seq_num 610) 2024-09-21T09:57:08.135 INFO:tasks.rados.rados.0.smithi033.stdout:1857: expect (ObjNum 605 snap 162 seq_num 605) 2024-09-21T09:57:08.180 INFO:tasks.rados.rados.0.smithi033.stdout:1858: expect (ObjNum 548 snap 148 seq_num 548) 2024-09-21T09:57:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1855: done (3 left) 2024-09-21T09:57:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1857: done (2 left) 2024-09-21T09:57:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1858: done (1 left) 2024-09-21T09:57:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1859: done (0 left) 2024-09-21T09:57:08.467 INFO:tasks.rados.rados.0.smithi033.stdout:1860: setattr oid 45 current snap is 163 2024-09-21T09:57:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 163 2024-09-21T09:57:08.468 INFO:tasks.rados.rados.0.smithi033.stdout:1861: seq_num 612 ranges {2392064=655360} 2024-09-21T09:57:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:1861: writing smithi03385152-4 from 2392064 to 3047424 tid 1 2024-09-21T09:57:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:1862: rollback oid 26 current snap is 163 2024-09-21T09:57:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 26 to 161 2024-09-21T09:57:08.485 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 955 (ObjNum 610 snap 162 seq_num 610) dirty exists 2024-09-21T09:57:08.486 INFO:tasks.rados.rados.0.smithi033.stdout:1860: done (2 left) 2024-09-21T09:57:08.486 INFO:tasks.rados.rados.0.smithi033.stdout:1863: copy_from oid 14 from oid 24 current snap is 163 2024-09-21T09:57:08.486 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 163 2024-09-21T09:57:08.486 INFO:tasks.rados.rados.0.smithi033.stdout:1864: seq_num 613 ranges {2007040=671744} 2024-09-21T09:57:08.500 INFO:tasks.rados.rados.0.smithi033.stdout:1864: writing smithi03385152-42 from 2007040 to 2678784 tid 1 2024-09-21T09:57:08.537 INFO:tasks.rados.rados.0.smithi033.stdout:1865: delete oid 40 current snap is 163 2024-09-21T09:57:08.537 INFO:tasks.rados.rados.0.smithi033.stdout:1863: finishing copy_from to smithi03385152-14 2024-09-21T09:57:08.537 INFO:tasks.rados.rados.0.smithi033.stdout:1863: got expected ENOENT (src dne) 2024-09-21T09:57:08.537 INFO:tasks.rados.rados.0.smithi033.stdout:1863: finishing copy_from racing read to smithi03385152-14 2024-09-21T09:57:08.537 INFO:tasks.rados.rados.0.smithi033.stdout:1862: finishing rollback tid 0 to smithi03385152-26 2024-09-21T09:57:08.537 INFO:tasks.rados.rados.0.smithi033.stdout:1862: finishing rollback tid 1 to smithi03385152-26 2024-09-21T09:57:08.538 INFO:tasks.rados.rados.0.smithi033.stdout:1862: finishing rollback tid 2 to smithi03385152-26 2024-09-21T09:57:08.538 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1089 (ObjNum 549 snap 148 seq_num 549) dirty exists 2024-09-21T09:57:08.538 INFO:tasks.rados.rados.0.smithi033.stdout:1861: finishing write tid 1 to smithi03385152-4 2024-09-21T09:57:08.538 INFO:tasks.rados.rados.0.smithi033.stdout:1861: finishing write tid 2 to smithi03385152-4 2024-09-21T09:57:08.538 INFO:tasks.rados.rados.0.smithi033.stdout:1861: finishing write tid 3 to smithi03385152-4 2024-09-21T09:57:08.538 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1019 (ObjNum 612 snap 163 seq_num 612) dirty exists 2024-09-21T09:57:08.538 INFO:tasks.rados.rados.0.smithi033.stdout:1861: left oid 4 (ObjNum 612 snap 163 seq_num 612) 2024-09-21T09:57:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:1864: finishing write tid 1 to smithi03385152-42 2024-09-21T09:57:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:1864: finishing write tid 2 to smithi03385152-42 2024-09-21T09:57:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:1861: done (4 left) 2024-09-21T09:57:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:1862: done (3 left) 2024-09-21T09:57:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:1863: done (2 left) 2024-09-21T09:57:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:1865: done (1 left) 2024-09-21T09:57:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:1866: read oid 26 snap -1 2024-09-21T09:57:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1866: expect (ObjNum 549 snap 148 seq_num 549) 2024-09-21T09:57:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1867: read oid 29 snap -1 2024-09-21T09:57:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1867: expect (ObjNum 529 snap 143 seq_num 529) 2024-09-21T09:57:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1868: copy_from oid 22 from oid 35 current snap is 163 2024-09-21T09:57:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1869: snap_create 2024-09-21T09:57:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1864: finishing write tid 3 to smithi03385152-42 2024-09-21T09:57:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 741 (ObjNum 613 snap 163 seq_num 613) dirty exists 2024-09-21T09:57:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:1864: left oid 42 (ObjNum 613 snap 163 seq_num 613) 2024-09-21T09:57:08.573 INFO:tasks.rados.rados.0.smithi033.stdout:1866: expect (ObjNum 549 snap 148 seq_num 549) 2024-09-21T09:57:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:08 smithi033 ceph-mon[98546]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:57:09.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:08 smithi033 ceph-mon[98546]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:57:09.008 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:08 smithi033 ceph-mon[96568]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:57:09.009 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:08 smithi033 ceph-mon[96568]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:57:09.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:08 smithi105 ceph-mon[86649]: osdmap e400: 8 total, 8 up, 8 in 2024-09-21T09:57:09.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:08 smithi105 ceph-mon[86649]: osdmap e401: 8 total, 8 up, 8 in 2024-09-21T09:57:09.107 INFO:tasks.rados.rados.0.smithi033.stdout:1867: expect (ObjNum 529 snap 143 seq_num 529) 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:1868: finishing copy_from racing read to smithi03385152-22 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:1868: finishing copy_from to smithi03385152-22 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 169 (ObjNum 574 snap 155 seq_num 574) dirty exists 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:1864: done (4 left) 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:1866: done (3 left) 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:1867: done (2 left) 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:1868: done (1 left) 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:1869: done (0 left) 2024-09-21T09:57:09.701 INFO:tasks.rados.rados.0.smithi033.stdout:1870: snap_create 2024-09-21T09:57:10.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:09 smithi033 ceph-mon[96568]: pgmap v230: 121 pgs: 121 active+clean; 163 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 5.2 MiB/s wr, 26 op/s 2024-09-21T09:57:10.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:09 smithi033 ceph-mon[96568]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:57:10.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:09 smithi033 ceph-mon[98546]: pgmap v230: 121 pgs: 121 active+clean; 163 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 5.2 MiB/s wr, 26 op/s 2024-09-21T09:57:10.001 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:09 smithi033 ceph-mon[98546]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:57:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:09 smithi105 ceph-mon[86649]: pgmap v230: 121 pgs: 121 active+clean; 163 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 5.2 MiB/s wr, 26 op/s 2024-09-21T09:57:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:09 smithi105 ceph-mon[86649]: osdmap e402: 8 total, 8 up, 8 in 2024-09-21T09:57:10.533 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:10.619 INFO:tasks.rados.rados.0.smithi033.stdout:1870: done (0 left) 2024-09-21T09:57:10.619 INFO:tasks.rados.rados.0.smithi033.stdout:1871: snap_create 2024-09-21T09:57:11.619 INFO:tasks.rados.rados.0.smithi033.stdout:1871: done (0 left) 2024-09-21T09:57:11.619 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 166 2024-09-21T09:57:11.619 INFO:tasks.rados.rados.0.smithi033.stdout:1872: seq_num 615 ranges {0=704512} 2024-09-21T09:57:11.623 INFO:tasks.rados.rados.0.smithi033.stdout:1872: writing smithi03385152-28 from 0 to 704512 tid 1 2024-09-21T09:57:11.623 INFO:tasks.rados.rados.0.smithi033.stdout:1873: snap_create 2024-09-21T09:57:11.628 INFO:tasks.rados.rados.0.smithi033.stdout:1872: finishing write tid 1 to smithi03385152-28 2024-09-21T09:57:11.628 INFO:tasks.rados.rados.0.smithi033.stdout:1872: finishing write tid 2 to smithi03385152-28 2024-09-21T09:57:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:1872: finishing write tid 3 to smithi03385152-28 2024-09-21T09:57:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 873 (ObjNum 615 snap 166 seq_num 615) dirty exists 2024-09-21T09:57:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:1872: left oid 28 (ObjNum 615 snap 166 seq_num 615) 2024-09-21T09:57:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:11 smithi033 ceph-mon[98546]: pgmap v233: 121 pgs: 121 active+clean; 165 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 949 KiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:57:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:11 smithi033 ceph-mon[98546]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:57:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:11 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:11 smithi033 ceph-mon[96568]: pgmap v233: 121 pgs: 121 active+clean; 165 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 949 KiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:57:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:11 smithi033 ceph-mon[96568]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:57:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:11 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:11.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:57:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:57:11] "GET /metrics HTTP/1.1" 200 36178 "" "Prometheus/2.51.0" 2024-09-21T09:57:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:11 smithi105 ceph-mon[86649]: pgmap v233: 121 pgs: 121 active+clean; 165 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 949 KiB/s rd, 1.2 MiB/s wr, 11 op/s 2024-09-21T09:57:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:11 smithi105 ceph-mon[86649]: osdmap e403: 8 total, 8 up, 8 in 2024-09-21T09:57:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:11 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:12.630 INFO:tasks.rados.rados.0.smithi033.stdout:1872: done (1 left) 2024-09-21T09:57:12.630 INFO:tasks.rados.rados.0.smithi033.stdout:1873: done (0 left) 2024-09-21T09:57:12.630 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 167 2024-09-21T09:57:12.630 INFO:tasks.rados.rados.0.smithi033.stdout:1874: seq_num 616 ranges {1867776=720896} 2024-09-21T09:57:12.644 INFO:tasks.rados.rados.0.smithi033.stdout:1874: writing smithi03385152-22 from 1867776 to 2588672 tid 1 2024-09-21T09:57:12.644 INFO:tasks.rados.rados.0.smithi033.stdout:1875: copy_from oid 50 from oid 15 current snap is 167 2024-09-21T09:57:12.644 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 167 2024-09-21T09:57:12.644 INFO:tasks.rados.rados.0.smithi033.stdout:1876: seq_num 617 ranges {0=737280} 2024-09-21T09:57:12.649 INFO:tasks.rados.rados.0.smithi033.stdout:1876: writing smithi03385152-8 from 0 to 737280 tid 1 2024-09-21T09:57:12.649 INFO:tasks.rados.rados.0.smithi033.stdout:1877: copy_from oid 31 from oid 27 current snap is 167 2024-09-21T09:57:12.649 INFO:tasks.rados.rados.0.smithi033.stdout:1878: snap_create 2024-09-21T09:57:12.649 INFO:tasks.rados.rados.0.smithi033.stdout:1875: finishing copy_from to smithi03385152-50 2024-09-21T09:57:12.649 INFO:tasks.rados.rados.0.smithi033.stdout:1875: got expected ENOENT (src dne) 2024-09-21T09:57:12.649 INFO:tasks.rados.rados.0.smithi033.stdout:1875: finishing copy_from racing read to smithi03385152-50 2024-09-21T09:57:12.650 INFO:tasks.rados.rados.0.smithi033.stdout:1874: finishing write tid 1 to smithi03385152-22 2024-09-21T09:57:12.651 INFO:tasks.rados.rados.0.smithi033.stdout:1874: finishing write tid 2 to smithi03385152-22 2024-09-21T09:57:12.653 INFO:tasks.rados.rados.0.smithi033.stdout:1874: finishing write tid 3 to smithi03385152-22 2024-09-21T09:57:12.653 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 960 (ObjNum 616 snap 167 seq_num 616) dirty exists 2024-09-21T09:57:12.653 INFO:tasks.rados.rados.0.smithi033.stdout:1874: left oid 22 (ObjNum 616 snap 167 seq_num 616) 2024-09-21T09:57:12.656 INFO:tasks.rados.rados.0.smithi033.stdout:1876: finishing write tid 1 to smithi03385152-8 2024-09-21T09:57:12.656 INFO:tasks.rados.rados.0.smithi033.stdout:1876: finishing write tid 2 to smithi03385152-8 2024-09-21T09:57:12.657 INFO:tasks.rados.rados.0.smithi033.stdout:1876: finishing write tid 3 to smithi03385152-8 2024-09-21T09:57:12.657 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 891 (ObjNum 617 snap 167 seq_num 617) dirty exists 2024-09-21T09:57:12.657 INFO:tasks.rados.rados.0.smithi033.stdout:1876: left oid 8 (ObjNum 617 snap 167 seq_num 617) 2024-09-21T09:57:12.661 INFO:tasks.rados.rados.0.smithi033.stdout:1877: finishing copy_from racing read to smithi03385152-31 2024-09-21T09:57:12.664 INFO:tasks.rados.rados.0.smithi033.stdout:1877: finishing copy_from to smithi03385152-31 2024-09-21T09:57:12.664 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 839 (ObjNum 511 snap 136 seq_num 511) dirty exists 2024-09-21T09:57:12.897 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:12 smithi033 ceph-mon[96568]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:57:12.898 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:12 smithi033 ceph-mon[98546]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:57:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:12 smithi105 ceph-mon[86649]: osdmap e404: 8 total, 8 up, 8 in 2024-09-21T09:57:13.474 INFO:tasks.rados.rados.0.smithi033.stdout:1874: done (4 left) 2024-09-21T09:57:13.474 INFO:tasks.rados.rados.0.smithi033.stdout:1875: done (3 left) 2024-09-21T09:57:13.474 INFO:tasks.rados.rados.0.smithi033.stdout:1876: done (2 left) 2024-09-21T09:57:13.474 INFO:tasks.rados.rados.0.smithi033.stdout:1877: done (1 left) 2024-09-21T09:57:13.474 INFO:tasks.rados.rados.0.smithi033.stdout:1878: done (0 left) 2024-09-21T09:57:13.474 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 168 2024-09-21T09:57:13.474 INFO:tasks.rados.rados.0.smithi033.stdout:1879: seq_num 618 ranges {0=753664} 2024-09-21T09:57:13.477 INFO:tasks.rados.rados.0.smithi033.stdout:1879: writing smithi03385152-43 from 0 to 753664 tid 1 2024-09-21T09:57:13.477 INFO:tasks.rados.rados.0.smithi033.stdout:1880: delete oid 18 current snap is 168 2024-09-21T09:57:13.483 INFO:tasks.rados.rados.0.smithi033.stdout:1880: done (1 left) 2024-09-21T09:57:13.483 INFO:tasks.rados.rados.0.smithi033.stdout:1881: read oid 13 snap -1 2024-09-21T09:57:13.483 INFO:tasks.rados.rados.0.smithi033.stdout:1881: expect deleted 2024-09-21T09:57:13.483 INFO:tasks.rados.rados.0.smithi033.stdout:1882: snap_create 2024-09-21T09:57:13.484 INFO:tasks.rados.rados.0.smithi033.stdout:1879: finishing write tid 1 to smithi03385152-43 2024-09-21T09:57:13.484 INFO:tasks.rados.rados.0.smithi033.stdout:1879: finishing write tid 2 to smithi03385152-43 2024-09-21T09:57:13.486 INFO:tasks.rados.rados.0.smithi033.stdout:1879: finishing write tid 3 to smithi03385152-43 2024-09-21T09:57:13.486 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 728 (ObjNum 618 snap 168 seq_num 618) dirty exists 2024-09-21T09:57:13.486 INFO:tasks.rados.rados.0.smithi033.stdout:1879: left oid 43 (ObjNum 618 snap 168 seq_num 618) 2024-09-21T09:57:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:13 smithi033 ceph-mon[98546]: pgmap v236: 121 pgs: 121 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 4.6 MiB/s wr, 30 op/s 2024-09-21T09:57:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:13 smithi033 ceph-mon[98546]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:57:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:13 smithi033 ceph-mon[98546]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:57:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:13 smithi033 ceph-mon[96568]: pgmap v236: 121 pgs: 121 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 4.6 MiB/s wr, 30 op/s 2024-09-21T09:57:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:13 smithi033 ceph-mon[96568]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:57:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:13 smithi033 ceph-mon[96568]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:57:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:13 smithi105 ceph-mon[86649]: pgmap v236: 121 pgs: 121 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 4.6 MiB/s wr, 30 op/s 2024-09-21T09:57:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:13 smithi105 ceph-mon[86649]: osdmap e405: 8 total, 8 up, 8 in 2024-09-21T09:57:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:13 smithi105 ceph-mon[86649]: osdmap e406: 8 total, 8 up, 8 in 2024-09-21T09:57:14.472 INFO:tasks.rados.rados.0.smithi033.stdout:1879: done (2 left) 2024-09-21T09:57:14.472 INFO:tasks.rados.rados.0.smithi033.stdout:1881: done (1 left) 2024-09-21T09:57:14.472 INFO:tasks.rados.rados.0.smithi033.stdout:1882: done (0 left) 2024-09-21T09:57:14.472 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 169 2024-09-21T09:57:14.472 INFO:tasks.rados.rados.0.smithi033.stdout:1883: seq_num 620 ranges {172032=794624} 2024-09-21T09:57:14.477 INFO:tasks.rados.rados.0.smithi033.stdout:1883: writing smithi03385152-7 from 172032 to 966656 tid 1 2024-09-21T09:57:14.477 INFO:tasks.rados.rados.0.smithi033.stdout:1884: read oid 43 snap -1 2024-09-21T09:57:14.477 INFO:tasks.rados.rados.0.smithi033.stdout:1884: expect (ObjNum 618 snap 168 seq_num 618) 2024-09-21T09:57:14.477 INFO:tasks.rados.rados.0.smithi033.stdout:1885: snap_remove snap 162 2024-09-21T09:57:14.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:14 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:14.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:15.477 INFO:tasks.rados.rados.0.smithi033.stdout:1885: done (2 left) 2024-09-21T09:57:15.477 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 169 2024-09-21T09:57:15.477 INFO:tasks.rados.rados.0.smithi033.stdout:1886: seq_num 621 ranges {2629632=409600,3039232=401408} 2024-09-21T09:57:15.494 INFO:tasks.rados.rados.0.smithi033.stdout:1886: writing smithi03385152-25 from 2629632 to 3039232 tid 1 2024-09-21T09:57:15.497 INFO:tasks.rados.rados.0.smithi033.stdout:1886: writing smithi03385152-25 from 3039232 to 3440640 tid 2 2024-09-21T09:57:15.497 INFO:tasks.rados.rados.0.smithi033.stdout:1887: snap_remove snap 163 2024-09-21T09:57:15.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:15 smithi105 ceph-mon[86649]: pgmap v239: 121 pgs: 121 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 18 op/s 2024-09-21T09:57:15.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:15 smithi105 ceph-mon[86649]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:57:15.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:15 smithi033 ceph-mon[98546]: pgmap v239: 121 pgs: 121 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 18 op/s 2024-09-21T09:57:15.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:15 smithi033 ceph-mon[98546]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:57:15.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:15 smithi033 ceph-mon[96568]: pgmap v239: 121 pgs: 121 active+clean; 179 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 3.4 MiB/s wr, 18 op/s 2024-09-21T09:57:15.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:15 smithi033 ceph-mon[96568]: osdmap e407: 8 total, 8 up, 8 in 2024-09-21T09:57:16.021 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:57:16.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:16.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:16.485 INFO:tasks.rados.rados.0.smithi033.stdout:1887: done (3 left) 2024-09-21T09:57:16.485 INFO:tasks.rados.rados.0.smithi033.stdout:1888: setattr oid 40 current snap is 169 2024-09-21T09:57:16.485 INFO:tasks.rados.rados.0.smithi033.stdout:1884: expect (ObjNum 618 snap 168 seq_num 618) 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:1883: finishing write tid 1 to smithi03385152-7 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:1883: finishing write tid 2 to smithi03385152-7 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:1883: finishing write tid 3 to smithi03385152-7 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 1170 (ObjNum 620 snap 169 seq_num 620) dirty exists 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:1883: left oid 7 (ObjNum 620 snap 169 seq_num 620) 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:1886: finishing write tid 1 to smithi03385152-25 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:1886: finishing write tid 2 to smithi03385152-25 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:1886: finishing write tid 3 to smithi03385152-25 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:1886: finishing write tid 4 to smithi03385152-25 2024-09-21T09:57:16.509 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 894 (ObjNum 621 snap 169 seq_num 621) dirty exists 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1886: left oid 25 (ObjNum 621 snap 169 seq_num 621) 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1883: done (3 left) 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1884: done (2 left) 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1886: done (1 left) 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1889: copy_from oid 22 from oid 29 current snap is 169 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1890: rmattr oid 38 current snap is 169 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1890: done (2 left) 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1891: rollback oid 28 current snap is 169 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 167 2024-09-21T09:57:16.510 INFO:tasks.rados.rados.0.smithi033.stdout:1892: snap_create 2024-09-21T09:57:16.514 INFO:tasks.rados.rados.0.smithi033.stdout:1891: finishing rollback tid 0 to smithi03385152-28 2024-09-21T09:57:16.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 742 (ObjNum 129 snap 0 seq_num 90745352) dirty exists 2024-09-21T09:57:16.515 INFO:tasks.rados.rados.0.smithi033.stdout:1891: finishing rollback tid 1 to smithi03385152-28 2024-09-21T09:57:16.515 INFO:tasks.rados.rados.0.smithi033.stdout:1891: finishing rollback tid 2 to smithi03385152-28 2024-09-21T09:57:16.515 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 876 (ObjNum 615 snap 166 seq_num 615) dirty exists 2024-09-21T09:57:16.550 INFO:tasks.rados.rados.0.smithi033.stdout:1889: finishing copy_from racing read to smithi03385152-22 2024-09-21T09:57:16.553 INFO:tasks.rados.rados.0.smithi033.stdout:1889: finishing copy_from to smithi03385152-22 2024-09-21T09:57:16.553 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 885 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:57:16.608 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[98546]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:57:16.608 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[98546]: Upgrade: osd.3 is safe to restart 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[96568]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:57:16.609 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:16 smithi033 ceph-mon[96568]: Upgrade: osd.3 is safe to restart 2024-09-21T09:57:16.624 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:57:16.624 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (4m) 19s ago 8m 20.1M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (3m) 2m ago 8m 82.4M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (10m) 19s ago 10m 16.0M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (10m) 2m ago 10m 19.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (3m) 2m ago 13m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (3m) 19s ago 15m 521M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (2m) 19s ago 15m 52.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (2m) 2m ago 13m 22.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (2m) 19s ago 13m 45.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (4m) 19s ago 8m 11.7M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (3m) 2m ago 7m 9915k - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (92s) 19s ago 13m 145M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (57s) 19s ago 12m 106M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (24s) 19s ago 12m 11.8M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 76dbf29565ba 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (12m) 19s ago 12m 120M 2481M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be ba92d47c6c05 2024-09-21T09:57:16.625 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (11m) 2m ago 11m 167M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:57:16.626 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (11m) 2m ago 11m 193M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:57:16.626 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (10m) 2m ago 10m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:57:16.626 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (10m) 2m ago 10m 140M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:57:16.626 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (3m) 2m ago 8m 44.0M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:57:16.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:16 smithi105 ceph-mon[86649]: osdmap e408: 8 total, 8 up, 8 in 2024-09-21T09:57:16.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:16.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:16.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-21T09:57:16.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:16 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-21T09:57:16.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:16 smithi105 ceph-mon[86649]: Upgrade: osd.3 is safe to restart 2024-09-21T09:57:16.984 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:57:16.985 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:57:16.985 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:57:16.985 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 5, 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 7, 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T09:57:16.986 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:57:16.987 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:57:17.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:16 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:17.283 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "8/20 daemons upgraded", 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:57:17.284 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:57:17.491 INFO:tasks.rados.rados.0.smithi033.stdout:1888: done (3 left) 2024-09-21T09:57:17.491 INFO:tasks.rados.rados.0.smithi033.stdout:1889: done (2 left) 2024-09-21T09:57:17.491 INFO:tasks.rados.rados.0.smithi033.stdout:1891: done (1 left) 2024-09-21T09:57:17.492 INFO:tasks.rados.rados.0.smithi033.stdout:1892: done (0 left) 2024-09-21T09:57:17.492 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 170 2024-09-21T09:57:17.492 INFO:tasks.rados.rados.0.smithi033.stdout:1893: seq_num 622 ranges {0=425984,425984=401408} 2024-09-21T09:57:17.493 INFO:tasks.rados.rados.0.smithi033.stdout:1893: writing smithi03385152-18 from 0 to 425984 tid 1 2024-09-21T09:57:17.496 INFO:tasks.rados.rados.0.smithi033.stdout:1893: writing smithi03385152-18 from 425984 to 827392 tid 2 2024-09-21T09:57:17.496 INFO:tasks.rados.rados.0.smithi033.stdout:1894: snap_create 2024-09-21T09:57:17.496 INFO:tasks.rados.rados.0.smithi033.stdout:1893: finishing write tid 1 to smithi03385152-18 2024-09-21T09:57:17.498 INFO:tasks.rados.rados.0.smithi033.stdout:1893: finishing write tid 2 to smithi03385152-18 2024-09-21T09:57:17.498 INFO:tasks.rados.rados.0.smithi033.stdout:1893: finishing write tid 3 to smithi03385152-18 2024-09-21T09:57:17.500 INFO:tasks.rados.rados.0.smithi033.stdout:1893: finishing write tid 4 to smithi03385152-18 2024-09-21T09:57:17.501 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1002 (ObjNum 622 snap 170 seq_num 622) dirty exists 2024-09-21T09:57:17.501 INFO:tasks.rados.rados.0.smithi033.stdout:1893: left oid 18 (ObjNum 622 snap 170 seq_num 622) 2024-09-21T09:57:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: from='client.44240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: pgmap v242: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 554 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:57:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:57:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/692107656' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: from='client.44240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: pgmap v242: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 554 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/692107656' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:17.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:17.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:57:17.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: from='client.44240 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: from='client.34332 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: pgmap v242: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 554 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:57:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: osdmap e409: 8 total, 8 up, 8 in 2024-09-21T09:57:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: from='client.34338 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/692107656' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:17.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:17.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-21T09:57:17.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:18.497 INFO:tasks.rados.rados.0.smithi033.stdout:1893: done (1 left) 2024-09-21T09:57:18.497 INFO:tasks.rados.rados.0.smithi033.stdout:1894: done (0 left) 2024-09-21T09:57:18.497 INFO:tasks.rados.rados.0.smithi033.stdout:1895: setattr oid 44 current snap is 171 2024-09-21T09:57:18.498 INFO:tasks.rados.rados.0.smithi033.stdout:1896: rollback oid 20 current snap is 171 2024-09-21T09:57:18.498 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 170 2024-09-21T09:57:18.498 INFO:tasks.rados.rados.0.smithi033.stdout:1897: snap_remove snap 166 2024-09-21T09:57:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:18 smithi033 ceph-mon[98546]: from='client.44264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:18.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:18 smithi033 ceph-mon[98546]: Upgrade: Updating osd.3 2024-09-21T09:57:18.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:18 smithi033 ceph-mon[98546]: Deploying daemon osd.3 on smithi033 2024-09-21T09:57:18.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:18 smithi033 ceph-mon[98546]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:57:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:18 smithi033 ceph-mon[96568]: from='client.44264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:18 smithi033 ceph-mon[96568]: Upgrade: Updating osd.3 2024-09-21T09:57:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:18 smithi033 ceph-mon[96568]: Deploying daemon osd.3 on smithi033 2024-09-21T09:57:18.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:18 smithi033 ceph-mon[96568]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:57:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:18 smithi105 ceph-mon[86649]: from='client.44264 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:18 smithi105 ceph-mon[86649]: Upgrade: Updating osd.3 2024-09-21T09:57:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:18 smithi105 ceph-mon[86649]: Deploying daemon osd.3 on smithi033 2024-09-21T09:57:18.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:18 smithi105 ceph-mon[86649]: osdmap e410: 8 total, 8 up, 8 in 2024-09-21T09:57:19.499 INFO:tasks.rados.rados.0.smithi033.stdout:1897: done (2 left) 2024-09-21T09:57:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:1898: copy_from oid 39 from oid 18 current snap is 171 2024-09-21T09:57:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:1896: finishing rollback tid 1 to smithi03385152-20 2024-09-21T09:57:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 886 (ObjNum 33 snap 0 seq_num 91310608) dirty dne 2024-09-21T09:57:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1020 (ObjNum 608 snap 162 seq_num 608) dirty exists 2024-09-21T09:57:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:1895: done (2 left) 2024-09-21T09:57:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:1896: done (1 left) 2024-09-21T09:57:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 171 2024-09-21T09:57:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:1899: seq_num 623 ranges {2834432=442368,3276800=401408} 2024-09-21T09:57:19.517 INFO:tasks.rados.rados.0.smithi033.stdout:1899: writing smithi03385152-23 from 2834432 to 3276800 tid 1 2024-09-21T09:57:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:1899: writing smithi03385152-23 from 3276800 to 3678208 tid 2 2024-09-21T09:57:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 171 2024-09-21T09:57:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:1898: finishing copy_from racing read to smithi03385152-39 2024-09-21T09:57:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:1900: seq_num 624 ranges {9879552=458752,10338304=401408} 2024-09-21T09:57:19.577 INFO:tasks.rados.rados.0.smithi033.stdout:1900: writing smithi03385152-30 from 9879552 to 10338304 tid 1 2024-09-21T09:57:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:1900: writing smithi03385152-30 from 10338304 to 10739712 tid 2 2024-09-21T09:57:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:1901: read oid 19 snap -1 2024-09-21T09:57:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:1901: expect (ObjNum 600 snap 161 seq_num 600) 2024-09-21T09:57:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:1902: read oid 1 snap -1 2024-09-21T09:57:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:1902: expect deleted 2024-09-21T09:57:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:1903: snap_remove snap 161 2024-09-21T09:57:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:19 smithi033 ceph-mon[98546]: pgmap v245: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 554 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:57:19.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:19 smithi033 ceph-mon[98546]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:57:19.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:19 smithi033 ceph-mon[96568]: pgmap v245: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 554 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:57:19.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:19 smithi033 ceph-mon[96568]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:57:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:19 smithi105 ceph-mon[86649]: pgmap v245: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 554 KiB/s rd, 1.4 MiB/s wr, 6 op/s 2024-09-21T09:57:19.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:19 smithi105 ceph-mon[86649]: osdmap e411: 8 total, 8 up, 8 in 2024-09-21T09:57:20.509 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:20.515 INFO:tasks.rados.rados.0.smithi033.stdout:1903: done (5 left) 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1904: rollback oid 41 current snap is 171 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 165 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1898: finishing copy_from to smithi03385152-39 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1002 (ObjNum 622 snap 170 seq_num 622) dirty exists 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1899: finishing write tid 1 to smithi03385152-23 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1899: finishing write tid 2 to smithi03385152-23 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1899: finishing write tid 3 to smithi03385152-23 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1899: finishing write tid 4 to smithi03385152-23 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 889 (ObjNum 623 snap 171 seq_num 623) dirty exists 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1899: left oid 23 (ObjNum 623 snap 171 seq_num 623) 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1900: finishing write tid 1 to smithi03385152-30 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1898: done (5 left) 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1899: done (4 left) 2024-09-21T09:57:20.516 INFO:tasks.rados.rados.0.smithi033.stdout:1902: done (3 left) 2024-09-21T09:57:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:1905: read oid 29 snap -1 2024-09-21T09:57:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:1905: expect (ObjNum 529 snap 143 seq_num 529) 2024-09-21T09:57:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 171 2024-09-21T09:57:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:1900: finishing write tid 2 to smithi03385152-30 2024-09-21T09:57:20.517 INFO:tasks.rados.rados.0.smithi033.stdout:1906: seq_num 625 ranges {0=475136,475136=401408} 2024-09-21T09:57:20.519 INFO:tasks.rados.rados.0.smithi033.stdout:1906: writing smithi03385152-13 from 0 to 475136 tid 1 2024-09-21T09:57:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:1906: writing smithi03385152-13 from 475136 to 876544 tid 2 2024-09-21T09:57:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 171 2024-09-21T09:57:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:1900: finishing write tid 3 to smithi03385152-30 2024-09-21T09:57:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:1900: finishing write tid 4 to smithi03385152-30 2024-09-21T09:57:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 1186 (ObjNum 624 snap 171 seq_num 624) dirty exists 2024-09-21T09:57:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:1900: left oid 30 (ObjNum 624 snap 171 seq_num 624) 2024-09-21T09:57:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:1901: expect (ObjNum 600 snap 161 seq_num 600) 2024-09-21T09:57:20.545 INFO:tasks.rados.rados.0.smithi033.stdout:1904: finishing rollback tid 1 to smithi03385152-41 2024-09-21T09:57:20.545 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 0 (ObjNum 641 snap 0 seq_num 1499157320) dirty dne 2024-09-21T09:57:20.545 INFO:tasks.rados.rados.0.smithi033.stdout:1907: seq_num 626 ranges {2113536=491520,2605056=401408} 2024-09-21T09:57:20.560 INFO:tasks.rados.rados.0.smithi033.stdout:1907: writing smithi03385152-11 from 2113536 to 2605056 tid 1 2024-09-21T09:57:20.563 INFO:tasks.rados.rados.0.smithi033.stdout:1907: writing smithi03385152-11 from 2605056 to 3006464 tid 2 2024-09-21T09:57:20.563 INFO:tasks.rados.rados.0.smithi033.stdout:1900: done (5 left) 2024-09-21T09:57:20.563 INFO:tasks.rados.rados.0.smithi033.stdout:1901: done (4 left) 2024-09-21T09:57:20.563 INFO:tasks.rados.rados.0.smithi033.stdout:1904: done (3 left) 2024-09-21T09:57:20.563 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 171 2024-09-21T09:57:20.563 INFO:tasks.rados.rados.0.smithi033.stdout:1908: seq_num 627 ranges {3678208=507904,4186112=401408} 2024-09-21T09:57:20.586 INFO:tasks.rados.rados.0.smithi033.stdout:1908: writing smithi03385152-23 from 3678208 to 4186112 tid 1 2024-09-21T09:57:20.588 INFO:tasks.rados.rados.0.smithi033.stdout:1908: writing smithi03385152-23 from 4186112 to 4587520 tid 2 2024-09-21T09:57:20.588 INFO:tasks.rados.rados.0.smithi033.stdout:1909: copy_from oid 5 from oid 35 current snap is 171 2024-09-21T09:57:20.588 INFO:tasks.rados.rados.0.smithi033.stdout:1906: finishing write tid 1 to smithi03385152-13 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:1907: finishing write tid 1 to smithi03385152-11 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:1907: finishing write tid 2 to smithi03385152-11 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:1907: finishing write tid 3 to smithi03385152-11 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:1907: finishing write tid 4 to smithi03385152-11 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 731 (ObjNum 626 snap 171 seq_num 626) dirty exists 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:1907: left oid 11 (ObjNum 626 snap 171 seq_num 626) 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:1907: done (4 left) 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 171 2024-09-21T09:57:20.589 INFO:tasks.rados.rados.0.smithi033.stdout:1905: expect (ObjNum 529 snap 143 seq_num 529) 2024-09-21T09:57:20.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:20 smithi105 ceph-mon[86649]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:57:20.884 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:20 smithi033 ceph-mon[98546]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:57:20.884 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:20 smithi033 ceph-mon[96568]: osdmap e412: 8 total, 8 up, 8 in 2024-09-21T09:57:20.884 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:20 smithi033 systemd[1]: Stopping Ceph osd.3 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:57:21.200 INFO:tasks.rados.rados.0.smithi033.stdout:1906: finishing write tid 2 to smithi03385152-13 2024-09-21T09:57:21.201 INFO:tasks.rados.rados.0.smithi033.stdout:1906: finishing write tid 3 to smithi03385152-13 2024-09-21T09:57:21.201 INFO:tasks.rados.rados.0.smithi033.stdout:1906: finishing write tid 4 to smithi03385152-13 2024-09-21T09:57:21.201 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 1189 (ObjNum 625 snap 171 seq_num 625) dirty exists 2024-09-21T09:57:21.201 INFO:tasks.rados.rados.0.smithi033.stdout:1906: left oid 13 (ObjNum 625 snap 171 seq_num 625) 2024-09-21T09:57:21.201 INFO:tasks.rados.rados.0.smithi033.stdout:1910: seq_num 628 ranges {9404416=524288,9928704=401408} 2024-09-21T09:57:21.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[62324]: 2024-09-21T09:57:20.880+0000 7fe8116b0640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:57:21.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[62324]: 2024-09-21T09:57:20.880+0000 7fe8116b0640 -1 osd.3 413 *** Got signal Terminated *** 2024-09-21T09:57:21.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:20 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[62324]: 2024-09-21T09:57:20.880+0000 7fe8116b0640 -1 osd.3 413 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:57:21.254 INFO:tasks.rados.rados.0.smithi033.stdout:1910: writing smithi03385152-21 from 9404416 to 9928704 tid 1 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1910: writing smithi03385152-21 from 9928704 to 10330112 tid 2 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1905: done (4 left) 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1906: done (3 left) 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1911: read oid 46 snap -1 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1911: expect deleted 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1912: read oid 4 snap -1 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1912: expect (ObjNum 612 snap 163 seq_num 612) 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 171 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1908: finishing write tid 1 to smithi03385152-23 2024-09-21T09:57:21.258 INFO:tasks.rados.rados.0.smithi033.stdout:1913: seq_num 629 ranges {1368064=540672,1908736=401408} 2024-09-21T09:57:21.268 INFO:tasks.rados.rados.0.smithi033.stdout:1913: writing smithi03385152-31 from 1368064 to 1908736 tid 1 2024-09-21T09:57:21.270 INFO:tasks.rados.rados.0.smithi033.stdout:1913: writing smithi03385152-31 from 1908736 to 2310144 tid 2 2024-09-21T09:57:21.270 INFO:tasks.rados.rados.0.smithi033.stdout:1914: snap_remove snap 164 2024-09-21T09:57:21.522 INFO:tasks.rados.rados.0.smithi033.stdout:1914: done (6 left) 2024-09-21T09:57:21.522 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 171 2024-09-21T09:57:21.522 INFO:tasks.rados.rados.0.smithi033.stdout:1915: seq_num 630 ranges {0=557056,557056=401408} 2024-09-21T09:57:21.526 INFO:tasks.rados.rados.0.smithi033.stdout:1915: writing smithi03385152-15 from 0 to 557056 tid 1 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1915: writing smithi03385152-15 from 557056 to 958464 tid 2 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1916: delete oid 33 current snap is 171 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1908: finishing write tid 2 to smithi03385152-23 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1908: finishing write tid 3 to smithi03385152-23 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1908: finishing write tid 4 to smithi03385152-23 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 892 (ObjNum 627 snap 171 seq_num 627) dirty exists 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1908: left oid 23 (ObjNum 627 snap 171 seq_num 627) 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1909: finishing copy_from racing read to smithi03385152-5 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1909: finishing copy_from to smithi03385152-5 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 169 (ObjNum 574 snap 155 seq_num 574) dirty exists 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1910: finishing write tid 1 to smithi03385152-21 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1910: finishing write tid 2 to smithi03385152-21 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1910: finishing write tid 3 to smithi03385152-21 2024-09-21T09:57:21.530 INFO:tasks.rados.rados.0.smithi033.stdout:1910: finishing write tid 4 to smithi03385152-21 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 895 (ObjNum 628 snap 171 seq_num 628) dirty exists 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:1910: left oid 21 (ObjNum 628 snap 171 seq_num 628) 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:1913: finishing write tid 1 to smithi03385152-31 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:1913: finishing write tid 2 to smithi03385152-31 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:1913: finishing write tid 3 to smithi03385152-31 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:1913: finishing write tid 4 to smithi03385152-31 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1201 (ObjNum 629 snap 171 seq_num 629) dirty exists 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:1913: left oid 31 (ObjNum 629 snap 171 seq_num 629) 2024-09-21T09:57:21.531 INFO:tasks.rados.rados.0.smithi033.stdout:1912: expect (ObjNum 612 snap 163 seq_num 612) 2024-09-21T09:57:21.651 INFO:tasks.rados.rados.0.smithi033.stdout:1908: done (7 left) 2024-09-21T09:57:21.651 INFO:tasks.rados.rados.0.smithi033.stdout:1909: done (6 left) 2024-09-21T09:57:21.651 INFO:tasks.rados.rados.0.smithi033.stdout:1910: done (5 left) 2024-09-21T09:57:21.651 INFO:tasks.rados.rados.0.smithi033.stdout:1911: done (4 left) 2024-09-21T09:57:21.651 INFO:tasks.rados.rados.0.smithi033.stdout:1912: done (3 left) 2024-09-21T09:57:21.651 INFO:tasks.rados.rados.0.smithi033.stdout:1913: done (2 left) 2024-09-21T09:57:21.651 INFO:tasks.rados.rados.0.smithi033.stdout:1916: done (1 left) 2024-09-21T09:57:21.652 INFO:tasks.rados.rados.0.smithi033.stdout:1917: setattr oid 32 current snap is 171 2024-09-21T09:57:21.652 INFO:tasks.rados.rados.0.smithi033.stdout:1918: rmattr oid 49 current snap is 171 2024-09-21T09:57:21.653 INFO:tasks.rados.rados.0.smithi033.stdout:1918: done (2 left) 2024-09-21T09:57:21.653 INFO:tasks.rados.rados.0.smithi033.stdout:1919: delete oid 16 current snap is 171 2024-09-21T09:57:21.655 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 173 (ObjNum 574 snap 155 seq_num 574) dirty exists 2024-09-21T09:57:21.656 INFO:tasks.rados.rados.0.smithi033.stdout:1917: done (2 left) 2024-09-21T09:57:21.656 INFO:tasks.rados.rados.0.smithi033.stdout:1919: done (1 left) 2024-09-21T09:57:21.656 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 171 2024-09-21T09:57:21.656 INFO:tasks.rados.rados.0.smithi033.stdout:1920: seq_num 633 ranges {753664=606208,1359872=401408} 2024-09-21T09:57:21.664 INFO:tasks.rados.rados.0.smithi033.stdout:1920: writing smithi03385152-43 from 753664 to 1359872 tid 1 2024-09-21T09:57:21.666 INFO:tasks.rados.rados.0.smithi033.stdout:1920: writing smithi03385152-43 from 1359872 to 1761280 tid 2 2024-09-21T09:57:21.666 INFO:tasks.rados.rados.0.smithi033.stdout:1921: read oid 46 snap -1 2024-09-21T09:57:21.666 INFO:tasks.rados.rados.0.smithi033.stdout:1921: expect deleted 2024-09-21T09:57:21.666 INFO:tasks.rados.rados.0.smithi033.stdout:1922: delete oid 34 current snap is 171 2024-09-21T09:57:21.667 INFO:tasks.rados.rados.0.smithi033.stdout:1920: finishing write tid 1 to smithi03385152-43 2024-09-21T09:57:21.668 INFO:tasks.rados.rados.0.smithi033.stdout:1920: finishing write tid 2 to smithi03385152-43 2024-09-21T09:57:21.668 INFO:tasks.rados.rados.0.smithi033.stdout:1920: finishing write tid 3 to smithi03385152-43 2024-09-21T09:57:21.669 INFO:tasks.rados.rados.0.smithi033.stdout:1921: done (3 left) 2024-09-21T09:57:21.669 INFO:tasks.rados.rados.0.smithi033.stdout:1922: done (2 left) 2024-09-21T09:57:21.669 INFO:tasks.rados.rados.0.smithi033.stdout:1923: snap_create 2024-09-21T09:57:21.669 INFO:tasks.rados.rados.0.smithi033.stdout:1920: finishing write tid 4 to smithi03385152-43 2024-09-21T09:57:21.669 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 734 (ObjNum 633 snap 171 seq_num 633) dirty exists 2024-09-21T09:57:21.669 INFO:tasks.rados.rados.0.smithi033.stdout:1920: left oid 43 (ObjNum 633 snap 171 seq_num 633) 2024-09-21T09:57:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:21 smithi105 ceph-mon[86649]: pgmap v248: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-21T09:57:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:21 smithi105 ceph-mon[86649]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:57:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:21 smithi105 ceph-mon[86649]: osd.3 marked itself down and dead 2024-09-21T09:57:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:21 smithi033 ceph-mon[98546]: pgmap v248: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-21T09:57:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:21 smithi033 ceph-mon[98546]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:57:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:21 smithi033 ceph-mon[98546]: osd.3 marked itself down and dead 2024-09-21T09:57:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:21 smithi033 ceph-mon[96568]: pgmap v248: 121 pgs: 121 active+clean; 184 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 4 op/s 2024-09-21T09:57:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:21 smithi033 ceph-mon[96568]: osdmap e413: 8 total, 8 up, 8 in 2024-09-21T09:57:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:21 smithi033 ceph-mon[96568]: osd.3 marked itself down and dead 2024-09-21T09:57:21.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:57:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:57:21] "GET /metrics HTTP/1.1" 200 36320 "" "Prometheus/2.51.0" 2024-09-21T09:57:22.366 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:22 smithi033 podman[117498]: 2024-09-21 09:57:22.05447226 +0000 UTC m=+1.278246144 container died ba92d47c6c054c6825e89625b6b1b3cf1694a41de7db54e0b504137c9b02cd17 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:57:22.530 INFO:tasks.rados.rados.0.smithi033.stdout:1920: done (2 left) 2024-09-21T09:57:22.530 INFO:tasks.rados.rados.0.smithi033.stdout:1923: done (1 left) 2024-09-21T09:57:22.531 INFO:tasks.rados.rados.0.smithi033.stdout:1924: snap_create 2024-09-21T09:57:22.542 INFO:tasks.rados.rados.0.smithi033.stdout:1915: finishing write tid 1 to smithi03385152-15 2024-09-21T09:57:22.543 INFO:tasks.rados.rados.0.smithi033.stdout:1915: finishing write tid 2 to smithi03385152-15 2024-09-21T09:57:22.543 INFO:tasks.rados.rados.0.smithi033.stdout:1915: finishing write tid 3 to smithi03385152-15 2024-09-21T09:57:22.547 INFO:tasks.rados.rados.0.smithi033.stdout:1915: finishing write tid 4 to smithi03385152-15 2024-09-21T09:57:22.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 894 (ObjNum 630 snap 171 seq_num 630) dirty exists 2024-09-21T09:57:22.547 INFO:tasks.rados.rados.0.smithi033.stdout:1915: left oid 15 (ObjNum 630 snap 171 seq_num 630) 2024-09-21T09:57:22.707 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:22 smithi033 ceph-mon[98546]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:57:22.707 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:22 smithi033 ceph-mon[98546]: osdmap e414: 8 total, 7 up, 8 in 2024-09-21T09:57:22.707 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:22 smithi033 podman[117498]: 2024-09-21 09:57:22.46272304 +0000 UTC m=+1.686496920 container cleanup ba92d47c6c054c6825e89625b6b1b3cf1694a41de7db54e0b504137c9b02cd17 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, RELEASE=quincy-fbd1a6a, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T09:57:22.707 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:22 smithi033 bash[117498]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3 2024-09-21T09:57:22.708 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:22 smithi033 ceph-mon[96568]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:57:22.708 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:22 smithi033 ceph-mon[96568]: osdmap e414: 8 total, 7 up, 8 in 2024-09-21T09:57:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:22 smithi105 ceph-mon[86649]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:57:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:22 smithi105 ceph-mon[86649]: osdmap e414: 8 total, 7 up, 8 in 2024-09-21T09:57:22.996 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:22 smithi033 podman[117518]: 2024-09-21 09:57:22.705733389 +0000 UTC m=+0.641245381 container remove ba92d47c6c054c6825e89625b6b1b3cf1694a41de7db54e0b504137c9b02cd17 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:57:23.285 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:23 smithi033 podman[117613]: 2024-09-21 09:57:22.925317143 +0000 UTC m=+0.016343449 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:57:23.286 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:23 smithi033 podman[117613]: 2024-09-21 09:57:23.040159517 +0000 UTC m=+0.131185856 container create 929ad95342d75af62815b695ff477f2dba10a186cea45cf059c0ba9509a8d35e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T09:57:23.469 INFO:tasks.rados.rados.0.smithi033.stdout:1915: done (1 left) 2024-09-21T09:57:23.469 INFO:tasks.rados.rados.0.smithi033.stdout:1924: done (0 left) 2024-09-21T09:57:23.469 INFO:tasks.rados.rados.0.smithi033.stdout:1925: read oid 37 snap 172 2024-09-21T09:57:23.469 INFO:tasks.rados.rados.0.smithi033.stdout:1925: expect (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:57:23.469 INFO:tasks.rados.rados.0.smithi033.stdout:1926: snap_create 2024-09-21T09:57:23.494 INFO:tasks.rados.rados.0.smithi033.stdout:1925: expect (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:57:23.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:23 smithi033 ceph-mon[98546]: pgmap v251: 121 pgs: 16 stale+active+clean, 105 active+clean; 196 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.5 MiB/s wr, 18 op/s 2024-09-21T09:57:23.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:23 smithi033 ceph-mon[98546]: osdmap e415: 8 total, 7 up, 8 in 2024-09-21T09:57:23.681 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:23 smithi033 ceph-mon[98546]: osdmap e416: 8 total, 7 up, 8 in 2024-09-21T09:57:23.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:23 smithi033 ceph-mon[96568]: pgmap v251: 121 pgs: 16 stale+active+clean, 105 active+clean; 196 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.5 MiB/s wr, 18 op/s 2024-09-21T09:57:23.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:23 smithi033 ceph-mon[96568]: osdmap e415: 8 total, 7 up, 8 in 2024-09-21T09:57:23.682 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:23 smithi033 ceph-mon[96568]: osdmap e416: 8 total, 7 up, 8 in 2024-09-21T09:57:23.682 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:23 smithi033 podman[117613]: 2024-09-21 09:57:23.284571483 +0000 UTC m=+0.375597783 container init 929ad95342d75af62815b695ff477f2dba10a186cea45cf059c0ba9509a8d35e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T09:57:23.682 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:23 smithi033 podman[117613]: 2024-09-21 09:57:23.290157643 +0000 UTC m=+0.381183937 container start 929ad95342d75af62815b695ff477f2dba10a186cea45cf059c0ba9509a8d35e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:57:23.682 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:23 smithi033 podman[117613]: 2024-09-21 09:57:23.375975426 +0000 UTC m=+0.467001737 container attach 929ad95342d75af62815b695ff477f2dba10a186cea45cf059c0ba9509a8d35e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:57:23.682 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:23 smithi033 podman[117613]: 2024-09-21 09:57:23.493568518 +0000 UTC m=+0.584594821 container died 929ad95342d75af62815b695ff477f2dba10a186cea45cf059c0ba9509a8d35e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:57:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:23 smithi105 ceph-mon[86649]: pgmap v251: 121 pgs: 16 stale+active+clean, 105 active+clean; 196 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 8.9 MiB/s rd, 3.5 MiB/s wr, 18 op/s 2024-09-21T09:57:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:23 smithi105 ceph-mon[86649]: osdmap e415: 8 total, 7 up, 8 in 2024-09-21T09:57:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:23 smithi105 ceph-mon[86649]: osdmap e416: 8 total, 7 up, 8 in 2024-09-21T09:57:24.472 INFO:tasks.rados.rados.0.smithi033.stdout:1925: done (1 left) 2024-09-21T09:57:24.473 INFO:tasks.rados.rados.0.smithi033.stdout:1926: done (0 left) 2024-09-21T09:57:24.473 INFO:tasks.rados.rados.0.smithi033.stdout:1927: snap_remove snap 169 2024-09-21T09:57:24.496 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 podman[117656]: 2024-09-21 09:57:24.138480439 +0000 UTC m=+0.635213106 container remove 929ad95342d75af62815b695ff477f2dba10a186cea45cf059c0ba9509a8d35e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:57:24.496 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.3.service: Deactivated successfully. 2024-09-21T09:57:24.496 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 systemd[1]: Stopped Ceph osd.3 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:57:24.496 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.3.service: Consumed 9.624s CPU time. 2024-09-21T09:57:24.496 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 systemd[1]: Starting Ceph osd.3 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:57:24.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:24.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:24.868 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 podman[117760]: 2024-09-21 09:57:24.500989552 +0000 UTC m=+0.016793693 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:57:24.868 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 podman[117760]: 2024-09-21 09:57:24.614401557 +0000 UTC m=+0.130205678 container create 8b801ebcf606379cd1a234cfa358f75baed37b1b5f7da7accd20e3f70de0b2be (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T09:57:25.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 podman[117760]: 2024-09-21 09:57:24.866923234 +0000 UTC m=+0.382727370 container init 8b801ebcf606379cd1a234cfa358f75baed37b1b5f7da7accd20e3f70de0b2be (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-21T09:57:25.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 podman[117760]: 2024-09-21 09:57:24.871866257 +0000 UTC m=+0.387670385 container start 8b801ebcf606379cd1a234cfa358f75baed37b1b5f7da7accd20e3f70de0b2be (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-62da54b, io.buildah.version=1.37.2) 2024-09-21T09:57:25.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:24 smithi033 podman[117760]: 2024-09-21 09:57:24.958546087 +0000 UTC m=+0.474350208 container attach 8b801ebcf606379cd1a234cfa358f75baed37b1b5f7da7accd20e3f70de0b2be (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-21T09:57:25.247 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:57:25.247 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:57:25.247 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:57:25.247 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:57:25.478 INFO:tasks.rados.rados.0.smithi033.stdout:1927: done (0 left) 2024-09-21T09:57:25.478 INFO:tasks.rados.rados.0.smithi033.stdout:1928: copy_from oid 10 from oid 48 current snap is 174 2024-09-21T09:57:25.478 INFO:tasks.rados.rados.0.smithi033.stdout:1929: snap_create 2024-09-21T09:57:25.496 INFO:tasks.rados.rados.0.smithi033.stdout:1928: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:57:25.497 INFO:tasks.rados.rados.0.smithi033.stdout:1928: finishing copy_from to smithi03385152-10 2024-09-21T09:57:25.497 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 853 (ObjNum 508 snap 135 seq_num 508) dirty exists 2024-09-21T09:57:25.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:25 smithi105 ceph-mon[86649]: pgmap v254: 121 pgs: 2 active+undersized, 15 stale+active+clean, 2 active+undersized+degraded, 102 active+clean; 198 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 4.1 MiB/s wr, 20 op/s; 2/321 objects degraded (0.623%) 2024-09-21T09:57:25.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:25 smithi105 ceph-mon[86649]: osdmap e417: 8 total, 7 up, 8 in 2024-09-21T09:57:25.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:25 smithi105 ceph-mon[86649]: Health check failed: Degraded data redundancy: 2/321 objects degraded (0.623%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:25.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-mon[96568]: pgmap v254: 121 pgs: 2 active+undersized, 15 stale+active+clean, 2 active+undersized+degraded, 102 active+clean; 198 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 4.1 MiB/s wr, 20 op/s; 2/321 objects degraded (0.623%) 2024-09-21T09:57:25.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-mon[96568]: osdmap e417: 8 total, 7 up, 8 in 2024-09-21T09:57:25.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-mon[96568]: Health check failed: Degraded data redundancy: 2/321 objects degraded (0.623%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-mon[98546]: pgmap v254: 121 pgs: 2 active+undersized, 15 stale+active+clean, 2 active+undersized+degraded, 102 active+clean; 198 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 4.1 MiB/s wr, 20 op/s; 2/321 objects degraded (0.623%) 2024-09-21T09:57:25.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-mon[98546]: osdmap e417: 8 total, 7 up, 8 in 2024-09-21T09:57:25.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-mon[98546]: Health check failed: Degraded data redundancy: 2/321 objects degraded (0.623%), 2 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:25.747 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:57:25.747 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:57:25.747 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:57:25.747 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:57:26.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:25.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:26.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:25.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate[117775]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 bash[117760]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-21T09:57:26.157 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:25 smithi033 podman[117760]: 2024-09-21 09:57:25.825970305 +0000 UTC m=+1.341774445 container died 8b801ebcf606379cd1a234cfa358f75baed37b1b5f7da7accd20e3f70de0b2be (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T09:57:26.485 INFO:tasks.rados.rados.0.smithi033.stdout:1928: done (1 left) 2024-09-21T09:57:26.486 INFO:tasks.rados.rados.0.smithi033.stdout:1929: done (0 left) 2024-09-21T09:57:26.486 INFO:tasks.rados.rados.0.smithi033.stdout:1930: read oid 11 snap -1 2024-09-21T09:57:26.486 INFO:tasks.rados.rados.0.smithi033.stdout:1930: expect (ObjNum 626 snap 171 seq_num 626) 2024-09-21T09:57:26.486 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 175 2024-09-21T09:57:26.486 INFO:tasks.rados.rados.0.smithi033.stdout:1931: seq_num 635 ranges {0=638976,638976=401408} 2024-09-21T09:57:26.489 INFO:tasks.rados.rados.0.smithi033.stdout:1931: writing smithi03385152-34 from 0 to 638976 tid 1 2024-09-21T09:57:26.493 INFO:tasks.rados.rados.0.smithi033.stdout:1931: writing smithi03385152-34 from 638976 to 1040384 tid 2 2024-09-21T09:57:26.493 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 175 2024-09-21T09:57:26.493 INFO:tasks.rados.rados.0.smithi033.stdout:1932: seq_num 636 ranges {3031040=655360,3686400=401408} 2024-09-21T09:57:26.513 INFO:tasks.rados.rados.0.smithi033.stdout:1932: writing smithi03385152-45 from 3031040 to 3686400 tid 1 2024-09-21T09:57:26.516 INFO:tasks.rados.rados.0.smithi033.stdout:1932: writing smithi03385152-45 from 3686400 to 4087808 tid 2 2024-09-21T09:57:26.516 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 175 2024-09-21T09:57:26.516 INFO:tasks.rados.rados.0.smithi033.stdout:1933: seq_num 637 ranges {3440640=671744,4112384=401408} 2024-09-21T09:57:26.538 INFO:tasks.rados.rados.0.smithi033.stdout:1933: writing smithi03385152-25 from 3440640 to 4112384 tid 1 2024-09-21T09:57:26.541 INFO:tasks.rados.rados.0.smithi033.stdout:1933: writing smithi03385152-25 from 4112384 to 4513792 tid 2 2024-09-21T09:57:26.541 INFO:tasks.rados.rados.0.smithi033.stdout:1934: rollback oid 6 current snap is 175 2024-09-21T09:57:26.541 INFO:tasks.rados.rados.0.smithi033.stdout:1930: expect (ObjNum 626 snap 171 seq_num 626) 2024-09-21T09:57:26.725 INFO:tasks.rados.rados.0.smithi033.stdout:1931: finishing write tid 1 to smithi03385152-34 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1931: finishing write tid 2 to smithi03385152-34 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1931: finishing write tid 3 to smithi03385152-34 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1931: finishing write tid 4 to smithi03385152-34 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 982 (ObjNum 635 snap 175 seq_num 635) dirty exists 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1931: left oid 34 (ObjNum 635 snap 175 seq_num 635) 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 174 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1932: finishing write tid 1 to smithi03385152-45 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1932: finishing write tid 2 to smithi03385152-45 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1932: finishing write tid 3 to smithi03385152-45 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1932: finishing write tid 4 to smithi03385152-45 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 986 (ObjNum 636 snap 175 seq_num 636) dirty exists 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1932: left oid 45 (ObjNum 636 snap 175 seq_num 636) 2024-09-21T09:57:26.726 INFO:tasks.rados.rados.0.smithi033.stdout:1933: finishing write tid 1 to smithi03385152-25 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1933: finishing write tid 2 to smithi03385152-25 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1933: finishing write tid 3 to smithi03385152-25 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1933: finishing write tid 4 to smithi03385152-25 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 897 (ObjNum 637 snap 175 seq_num 637) dirty exists 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1933: left oid 25 (ObjNum 637 snap 175 seq_num 637) 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1930: done (4 left) 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1931: done (3 left) 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1932: done (2 left) 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1933: done (1 left) 2024-09-21T09:57:26.727 INFO:tasks.rados.rados.0.smithi033.stdout:1935: setattr oid 2 current snap is 175 2024-09-21T09:57:26.728 INFO:tasks.rados.rados.0.smithi033.stdout:1936: read oid 43 snap 160 2024-09-21T09:57:26.728 INFO:tasks.rados.rados.0.smithi033.stdout:1936: expect deleted 2024-09-21T09:57:26.728 INFO:tasks.rados.rados.0.smithi033.stdout:1937: read oid 39 snap -1 2024-09-21T09:57:26.728 INFO:tasks.rados.rados.0.smithi033.stdout:1937: expect (ObjNum 622 snap 170 seq_num 622) 2024-09-21T09:57:26.728 INFO:tasks.rados.rados.0.smithi033.stdout:1938: delete oid 30 current snap is 175 2024-09-21T09:57:26.728 INFO:tasks.rados.rados.0.smithi033.stdout:1934: finishing rollback tid 0 to smithi03385152-6 2024-09-21T09:57:26.729 INFO:tasks.rados.rados.0.smithi033.stdout:1934: finishing rollback tid 1 to smithi03385152-6 2024-09-21T09:57:26.729 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 898 (ObjNum 33 snap 0 seq_num 91513328) dirty exists 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1935: done (4 left) 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1936: done (3 left) 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1938: done (2 left) 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1939: rollback oid 45 current snap is 175 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 45 to 160 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1940: read oid 13 snap -1 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1940: expect (ObjNum 625 snap 171 seq_num 625) 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1941: read oid 18 snap -1 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1941: expect (ObjNum 622 snap 170 seq_num 622) 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:1942: rollback oid 33 current snap is 175 2024-09-21T09:57:26.730 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 173 2024-09-21T09:57:26.731 INFO:tasks.rados.rados.0.smithi033.stdout:1943: read oid 25 snap -1 2024-09-21T09:57:26.731 INFO:tasks.rados.rados.0.smithi033.stdout:1943: expect (ObjNum 637 snap 175 seq_num 637) 2024-09-21T09:57:26.731 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 175 2024-09-21T09:57:26.731 INFO:tasks.rados.rados.0.smithi033.stdout:1944: seq_num 639 ranges {0=704512,704512=401408} 2024-09-21T09:57:26.734 INFO:tasks.rados.rados.0.smithi033.stdout:1944: writing smithi03385152-40 from 0 to 704512 tid 1 2024-09-21T09:57:26.737 INFO:tasks.rados.rados.0.smithi033.stdout:1944: writing smithi03385152-40 from 704512 to 1105920 tid 2 2024-09-21T09:57:26.737 INFO:tasks.rados.rados.0.smithi033.stdout:1945: read oid 37 snap -1 2024-09-21T09:57:26.737 INFO:tasks.rados.rados.0.smithi033.stdout:1945: expect (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:57:26.738 INFO:tasks.rados.rados.0.smithi033.stdout:1946: snap_remove snap 171 2024-09-21T09:57:26.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:26 smithi033 ceph-mon[96568]: osdmap e418: 8 total, 7 up, 8 in 2024-09-21T09:57:26.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:26 smithi033 ceph-mon[98546]: osdmap e418: 8 total, 7 up, 8 in 2024-09-21T09:57:26.746 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:26 smithi033 podman[117936]: 2024-09-21 09:57:26.445765395 +0000 UTC m=+0.610336093 container remove 8b801ebcf606379cd1a234cfa358f75baed37b1b5f7da7accd20e3f70de0b2be (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-activate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T09:57:26.746 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:26 smithi033 podman[117999]: 2024-09-21 09:57:26.575610148 +0000 UTC m=+0.016843571 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:57:26.746 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:26 smithi033 podman[117999]: 2024-09-21 09:57:26.688274585 +0000 UTC m=+0.129507998 container create b85b61817088d7f3196a548dee4fa9fc186cdd295c5435d2ed988bd4187c85e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T09:57:26.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:26 smithi105 ceph-mon[86649]: osdmap e418: 8 total, 7 up, 8 in 2024-09-21T09:57:27.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:26 smithi033 podman[117999]: 2024-09-21 09:57:26.949369129 +0000 UTC m=+0.390602549 container init b85b61817088d7f3196a548dee4fa9fc186cdd295c5435d2ed988bd4187c85e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:57:27.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:26 smithi033 podman[117999]: 2024-09-21 09:57:26.954520161 +0000 UTC m=+0.395753572 container start b85b61817088d7f3196a548dee4fa9fc186cdd295c5435d2ed988bd4187c85e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:57:27.247 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:26 smithi033 ceph-osd[118037]: -- 172.21.15.33:0/3225964171 <== mon.0 v2:172.21.15.33:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558592b17a40 con 0x558592af5c00 2024-09-21T09:57:27.247 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:27 smithi033 bash[117999]: b85b61817088d7f3196a548dee4fa9fc186cdd295c5435d2ed988bd4187c85e7 2024-09-21T09:57:27.247 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:27 smithi033 systemd[1]: Started Ceph osd.3 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:57:27.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:26.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:27.493 INFO:tasks.rados.rados.0.smithi033.stdout:1946: done (9 left) 2024-09-21T09:57:27.494 INFO:tasks.rados.rados.0.smithi033.stdout:1947: rmattr oid 26 current snap is 175 2024-09-21T09:57:27.494 INFO:tasks.rados.rados.0.smithi033.stdout:1947: done (9 left) 2024-09-21T09:57:27.494 INFO:tasks.rados.rados.0.smithi033.stdout:1948: snap_remove snap 165 2024-09-21T09:57:27.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[98546]: pgmap v257: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 202 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 21/336 objects degraded (6.250%) 2024-09-21T09:57:27.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[98546]: osdmap e419: 8 total, 7 up, 8 in 2024-09-21T09:57:27.736 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:27.737 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:27.737 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:27.737 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[96568]: pgmap v257: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 202 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 21/336 objects degraded (6.250%) 2024-09-21T09:57:27.737 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[96568]: osdmap e419: 8 total, 7 up, 8 in 2024-09-21T09:57:27.737 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:27.737 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:27.737 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:27 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:27 smithi105 ceph-mon[86649]: pgmap v257: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 202 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.5 MiB/s wr, 12 op/s; 21/336 objects degraded (6.250%) 2024-09-21T09:57:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:27 smithi105 ceph-mon[86649]: osdmap e419: 8 total, 7 up, 8 in 2024-09-21T09:57:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:27 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:28.471 INFO:tasks.rados.rados.0.smithi033.stdout:1948: done (9 left) 2024-09-21T09:57:28.471 INFO:tasks.rados.rados.0.smithi033.stdout:1949: rollback oid 46 current snap is 175 2024-09-21T09:57:28.471 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 46 to 167 2024-09-21T09:57:28.472 INFO:tasks.rados.rados.0.smithi033.stdout:1934: finishing rollback tid 2 to smithi03385152-6 2024-09-21T09:57:28.472 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1039 (ObjNum 581 snap 156 seq_num 581) dirty exists 2024-09-21T09:57:28.472 INFO:tasks.rados.rados.0.smithi033.stdout:1937: expect (ObjNum 622 snap 170 seq_num 622) 2024-09-21T09:57:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:1942: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:57:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1036 (ObjNum 33 snap 0 seq_num 72668536) dirty dne 2024-09-21T09:57:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:1939: finishing rollback tid 0 to smithi03385152-45 2024-09-21T09:57:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:1939: finishing rollback tid 1 to smithi03385152-45 2024-09-21T09:57:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:1941: expect (ObjNum 622 snap 170 seq_num 622) 2024-09-21T09:57:28.543 INFO:tasks.rados.rados.0.smithi033.stdout:1939: finishing rollback tid 2 to smithi03385152-45 2024-09-21T09:57:28.543 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 989 (ObjNum 573 snap 155 seq_num 573) dirty exists 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1944: finishing write tid 1 to smithi03385152-40 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1944: finishing write tid 2 to smithi03385152-40 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1944: finishing write tid 3 to smithi03385152-40 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1934: done (9 left) 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1937: done (8 left) 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1939: done (7 left) 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1941: done (6 left) 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1942: done (5 left) 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 175 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1944: finishing write tid 4 to smithi03385152-40 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 745 (ObjNum 639 snap 175 seq_num 639) dirty exists 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1944: left oid 40 (ObjNum 639 snap 175 seq_num 639) 2024-09-21T09:57:28.544 INFO:tasks.rados.rados.0.smithi033.stdout:1940: expect (ObjNum 625 snap 171 seq_num 625) 2024-09-21T09:57:28.567 INFO:tasks.rados.rados.0.smithi033.stdout:1943: expect (ObjNum 637 snap 175 seq_num 637) 2024-09-21T09:57:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:28 smithi033 ceph-mon[98546]: osdmap e420: 8 total, 7 up, 8 in 2024-09-21T09:57:28.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:28 smithi033 ceph-mon[98546]: osdmap e421: 8 total, 7 up, 8 in 2024-09-21T09:57:28.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:28 smithi033 ceph-mon[96568]: osdmap e420: 8 total, 7 up, 8 in 2024-09-21T09:57:28.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:28 smithi033 ceph-mon[96568]: osdmap e421: 8 total, 7 up, 8 in 2024-09-21T09:57:28.747 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:28 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T09:57:28.352+0000 7f634f4c5640 -1 Falling back to public interface 2024-09-21T09:57:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:28 smithi105 ceph-mon[86649]: osdmap e420: 8 total, 7 up, 8 in 2024-09-21T09:57:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:28 smithi105 ceph-mon[86649]: osdmap e421: 8 total, 7 up, 8 in 2024-09-21T09:57:28.811 INFO:tasks.rados.rados.0.smithi033.stdout:1945: expect (ObjNum 503 snap 132 seq_num 503) 2024-09-21T09:57:29.203 INFO:tasks.rados.rados.0.smithi033.stdout:1949: finishing rollback tid 1 to smithi03385152-46 2024-09-21T09:57:29.203 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 0 (ObjNum 1441 snap 0 seq_num 91094304) dirty dne 2024-09-21T09:57:29.203 INFO:tasks.rados.rados.0.smithi033.stdout:1950: seq_num 640 ranges {704512=729088,1433600=401408} 2024-09-21T09:57:29.211 INFO:tasks.rados.rados.0.smithi033.stdout:1950: writing smithi03385152-28 from 704512 to 1433600 tid 1 2024-09-21T09:57:29.214 INFO:tasks.rados.rados.0.smithi033.stdout:1950: writing smithi03385152-28 from 1433600 to 1835008 tid 2 2024-09-21T09:57:29.214 INFO:tasks.rados.rados.0.smithi033.stdout:1940: done (5 left) 2024-09-21T09:57:29.214 INFO:tasks.rados.rados.0.smithi033.stdout:1943: done (4 left) 2024-09-21T09:57:29.214 INFO:tasks.rados.rados.0.smithi033.stdout:1944: done (3 left) 2024-09-21T09:57:29.214 INFO:tasks.rados.rados.0.smithi033.stdout:1945: done (2 left) 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:1949: done (1 left) 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:1951: rollback oid 6 current snap is 175 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 160 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:1952: rollback oid 9 current snap is 175 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 9 to 170 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:1953: copy_from oid 7 from oid 34 current snap is 175 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:1954: read oid 46 snap -1 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:1954: expect deleted 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:1955: delete oid 44 current snap is 175 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:1952: finishing rollback tid 1 to smithi03385152-9 2024-09-21T09:57:29.215 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 0 (ObjNum 529 snap 0 seq_num 91683232) dirty dne 2024-09-21T09:57:29.217 INFO:tasks.rados.rados.0.smithi033.stdout:1950: finishing write tid 1 to smithi03385152-28 2024-09-21T09:57:29.217 INFO:tasks.rados.rados.0.smithi033.stdout:1951: finishing rollback tid 0 to smithi03385152-6 2024-09-21T09:57:29.218 INFO:tasks.rados.rados.0.smithi033.stdout:1950: finishing write tid 2 to smithi03385152-28 2024-09-21T09:57:29.218 INFO:tasks.rados.rados.0.smithi033.stdout:1952: done (5 left) 2024-09-21T09:57:29.218 INFO:tasks.rados.rados.0.smithi033.stdout:1954: done (4 left) 2024-09-21T09:57:29.219 INFO:tasks.rados.rados.0.smithi033.stdout:1955: done (3 left) 2024-09-21T09:57:29.219 INFO:tasks.rados.rados.0.smithi033.stdout:1956: read oid 43 snap -1 2024-09-21T09:57:29.219 INFO:tasks.rados.rados.0.smithi033.stdout:1956: expect (ObjNum 633 snap 171 seq_num 633) 2024-09-21T09:57:29.219 INFO:tasks.rados.rados.0.smithi033.stdout:1957: snap_create 2024-09-21T09:57:29.219 INFO:tasks.rados.rados.0.smithi033.stdout:1951: finishing rollback tid 1 to smithi03385152-6 2024-09-21T09:57:29.219 INFO:tasks.rados.rados.0.smithi033.stdout:1950: finishing write tid 3 to smithi03385152-28 2024-09-21T09:57:29.219 INFO:tasks.rados.rados.0.smithi033.stdout:1951: finishing rollback tid 2 to smithi03385152-6 2024-09-21T09:57:29.219 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1042 (ObjNum 587 snap 158 seq_num 587) dirty exists 2024-09-21T09:57:29.221 INFO:tasks.rados.rados.0.smithi033.stdout:1950: finishing write tid 4 to smithi03385152-28 2024-09-21T09:57:29.221 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 879 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:57:29.221 INFO:tasks.rados.rados.0.smithi033.stdout:1950: left oid 28 (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:57:29.222 INFO:tasks.rados.rados.0.smithi033.stdout:1953: finishing copy_from racing read to smithi03385152-7 2024-09-21T09:57:29.225 INFO:tasks.rados.rados.0.smithi033.stdout:1956: expect (ObjNum 633 snap 171 seq_num 633) 2024-09-21T09:57:29.289 INFO:tasks.rados.rados.0.smithi033.stdout:1953: finishing copy_from to smithi03385152-7 2024-09-21T09:57:29.289 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 982 (ObjNum 635 snap 175 seq_num 635) dirty exists 2024-09-21T09:57:29.511 INFO:tasks.rados.rados.0.smithi033.stdout:1950: done (4 left) 2024-09-21T09:57:29.511 INFO:tasks.rados.rados.0.smithi033.stdout:1951: done (3 left) 2024-09-21T09:57:29.512 INFO:tasks.rados.rados.0.smithi033.stdout:1953: done (2 left) 2024-09-21T09:57:29.512 INFO:tasks.rados.rados.0.smithi033.stdout:1956: done (1 left) 2024-09-21T09:57:29.512 INFO:tasks.rados.rados.0.smithi033.stdout:1957: done (0 left) 2024-09-21T09:57:29.512 INFO:tasks.rados.rados.0.smithi033.stdout:1958: snap_remove snap 160 2024-09-21T09:57:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:29 smithi105 ceph-mon[86649]: pgmap v260: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 202 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 924 KiB/s wr, 5 op/s; 21/336 objects degraded (6.250%) 2024-09-21T09:57:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:29 smithi033 ceph-mon[98546]: pgmap v260: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 202 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 924 KiB/s wr, 5 op/s; 21/336 objects degraded (6.250%) 2024-09-21T09:57:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:29 smithi033 ceph-mon[96568]: pgmap v260: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 202 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 924 KiB/s wr, 5 op/s; 21/336 objects degraded (6.250%) 2024-09-21T09:57:30.515 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:30.521 INFO:tasks.rados.rados.0.smithi033.stdout:1958: done (0 left) 2024-09-21T09:57:30.521 INFO:tasks.rados.rados.0.smithi033.stdout:1959: snap_remove snap 172 2024-09-21T09:57:30.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:30 smithi033 ceph-mon[98546]: osdmap e422: 8 total, 7 up, 8 in 2024-09-21T09:57:30.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:30 smithi033 ceph-mon[96568]: osdmap e422: 8 total, 7 up, 8 in 2024-09-21T09:57:30.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:30 smithi105 ceph-mon[86649]: osdmap e422: 8 total, 7 up, 8 in 2024-09-21T09:57:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:1959: done (0 left) 2024-09-21T09:57:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 176 2024-09-21T09:57:31.522 INFO:tasks.rados.rados.0.smithi033.stdout:1960: seq_num 642 ranges {737280=761856,1499136=401408} 2024-09-21T09:57:31.531 INFO:tasks.rados.rados.0.smithi033.stdout:1960: writing smithi03385152-8 from 737280 to 1499136 tid 1 2024-09-21T09:57:31.535 INFO:tasks.rados.rados.0.smithi033.stdout:1960: writing smithi03385152-8 from 1499136 to 1900544 tid 2 2024-09-21T09:57:31.535 INFO:tasks.rados.rados.0.smithi033.stdout:1961: rollback oid 45 current snap is 176 2024-09-21T09:57:31.535 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 45 to 174 2024-09-21T09:57:31.535 INFO:tasks.rados.rados.0.smithi033.stdout:1962: rollback oid 9 current snap is 176 2024-09-21T09:57:31.535 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 9 to 174 2024-09-21T09:57:31.536 INFO:tasks.rados.rados.0.smithi033.stdout:1963: read oid 36 snap -1 2024-09-21T09:57:31.536 INFO:tasks.rados.rados.0.smithi033.stdout:1963: expect (ObjNum 592 snap 159 seq_num 592) 2024-09-21T09:57:31.536 INFO:tasks.rados.rados.0.smithi033.stdout:1964: snap_remove snap 159 2024-09-21T09:57:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: pgmap v263: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 204 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 592 KiB/s wr, 6 op/s; 22/342 objects degraded (6.433%) 2024-09-21T09:57:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 22/342 objects degraded (6.433%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: osdmap e423: 8 total, 7 up, 8 in 2024-09-21T09:57:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.785 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:57:31] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.51.0" 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: pgmap v263: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 204 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 592 KiB/s wr, 6 op/s; 22/342 objects degraded (6.433%) 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 22/342 objects degraded (6.433%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: osdmap e423: 8 total, 7 up, 8 in 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.786 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: pgmap v263: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 204 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 9.4 MiB/s rd, 592 KiB/s wr, 6 op/s; 22/342 objects degraded (6.433%) 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 22/342 objects degraded (6.433%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: osdmap e423: 8 total, 7 up, 8 in 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:31.787 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:32.060 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T09:57:31.937+0000 7f634f4c5640 -1 osd.3 413 log_to_monitors true 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1964: done (4 left) 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1965: delete oid 44 current snap is 176 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1962: finishing rollback tid 1 to smithi03385152-9 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1960: finishing write tid 1 to smithi03385152-8 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1960: finishing write tid 2 to smithi03385152-8 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1960: finishing write tid 3 to smithi03385152-8 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1961: finishing rollback tid 0 to smithi03385152-45 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1961: finishing rollback tid 1 to smithi03385152-45 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1960: finishing write tid 4 to smithi03385152-8 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 897 (ObjNum 642 snap 176 seq_num 642) dirty exists 2024-09-21T09:57:32.534 INFO:tasks.rados.rados.0.smithi033.stdout:1960: left oid 8 (ObjNum 642 snap 176 seq_num 642) 2024-09-21T09:57:32.535 INFO:tasks.rados.rados.0.smithi033.stdout:1961: finishing rollback tid 2 to smithi03385152-45 2024-09-21T09:57:32.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1007 (ObjNum 610 snap 162 seq_num 610) dirty exists 2024-09-21T09:57:32.535 INFO:tasks.rados.rados.0.smithi033.stdout:1963: expect (ObjNum 592 snap 159 seq_num 592) 2024-09-21T09:57:32.639 INFO:tasks.rados.rados.0.smithi033.stdout:1960: done (4 left) 2024-09-21T09:57:32.639 INFO:tasks.rados.rados.0.smithi033.stdout:1961: done (3 left) 2024-09-21T09:57:32.639 INFO:tasks.rados.rados.0.smithi033.stdout:1962: done (2 left) 2024-09-21T09:57:32.640 INFO:tasks.rados.rados.0.smithi033.stdout:1963: done (1 left) 2024-09-21T09:57:32.640 INFO:tasks.rados.rados.0.smithi033.stdout:1965: done (0 left) 2024-09-21T09:57:32.640 INFO:tasks.rados.rados.0.smithi033.stdout:1966: snap_create 2024-09-21T09:57:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:32 smithi105 ceph-mon[86649]: osdmap e424: 8 total, 7 up, 8 in 2024-09-21T09:57:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:32 smithi105 ceph-mon[86649]: from='osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:57:32.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:32 smithi105 ceph-mon[86649]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:57:32.856 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:32 smithi033 ceph-mon[98546]: osdmap e424: 8 total, 7 up, 8 in 2024-09-21T09:57:32.856 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:32 smithi033 ceph-mon[98546]: from='osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:57:32.856 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:32 smithi033 ceph-mon[98546]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:57:32.857 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 09:57:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T09:57:32.566+0000 7f634726f640 -1 osd.3 413 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:57:32.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:32 smithi033 ceph-mon[96568]: osdmap e424: 8 total, 7 up, 8 in 2024-09-21T09:57:32.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:32 smithi033 ceph-mon[96568]: from='osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:57:32.857 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:32 smithi033 ceph-mon[96568]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-21T09:57:33.535 INFO:tasks.rados.rados.0.smithi033.stdout:1966: done (0 left) 2024-09-21T09:57:33.535 INFO:tasks.rados.rados.0.smithi033.stdout:1967: delete oid 34 current snap is 177 2024-09-21T09:57:33.546 INFO:tasks.rados.rados.0.smithi033.stdout:1967: done (0 left) 2024-09-21T09:57:33.546 INFO:tasks.rados.rados.0.smithi033.stdout:1968: snap_remove snap 170 2024-09-21T09:57:33.915 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[98546]: pgmap v266: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 212 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 23/360 objects degraded (6.389%) 2024-09-21T09:57:33.915 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[98546]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:57:33.915 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[98546]: osdmap e425: 8 total, 7 up, 8 in 2024-09-21T09:57:33.915 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[98546]: from='osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:57:33.916 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[98546]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:57:33.916 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[96568]: pgmap v266: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 212 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 23/360 objects degraded (6.389%) 2024-09-21T09:57:33.916 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[96568]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:57:33.916 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[96568]: osdmap e425: 8 total, 7 up, 8 in 2024-09-21T09:57:33.916 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[96568]: from='osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:57:33.916 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:33 smithi033 ceph-mon[96568]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:57:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:33 smithi105 ceph-mon[86649]: pgmap v266: 121 pgs: 33 active+undersized, 8 active+undersized+degraded, 80 active+clean; 212 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 19 op/s; 23/360 objects degraded (6.389%) 2024-09-21T09:57:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:33 smithi105 ceph-mon[86649]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-21T09:57:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:33 smithi105 ceph-mon[86649]: osdmap e425: 8 total, 7 up, 8 in 2024-09-21T09:57:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:33 smithi105 ceph-mon[86649]: from='osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:57:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:33 smithi105 ceph-mon[86649]: from='osd.3 ' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi033", "root=default"]} : dispatch 2024-09-21T09:57:34.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:34.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:57:34.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:34.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:34.532 INFO:tasks.rados.rados.0.smithi033.stdout:1968: done (0 left) 2024-09-21T09:57:34.532 INFO:tasks.rados.rados.0.smithi033.stdout:1969: copy_from oid 3 from oid 28 current snap is 177 2024-09-21T09:57:34.532 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 177 2024-09-21T09:57:34.532 INFO:tasks.rados.rados.0.smithi033.stdout:1970: seq_num 645 ranges {3031040=409600,3440640=778240,4218880=24576} 2024-09-21T09:57:34.553 INFO:tasks.rados.rados.0.smithi033.stdout:1970: writing smithi03385152-45 from 3031040 to 3440640 tid 1 2024-09-21T09:57:34.558 INFO:tasks.rados.rados.0.smithi033.stdout:1970: writing smithi03385152-45 from 3440640 to 4218880 tid 2 2024-09-21T09:57:34.559 INFO:tasks.rados.rados.0.smithi033.stdout:1970: writing smithi03385152-45 from 4218880 to 4243456 tid 3 2024-09-21T09:57:34.559 INFO:tasks.rados.rados.0.smithi033.stdout:1971: read oid 14 snap -1 2024-09-21T09:57:34.559 INFO:tasks.rados.rados.0.smithi033.stdout:1971: expect (ObjNum 609 snap 162 seq_num 609) 2024-09-21T09:57:34.559 INFO:tasks.rados.rados.0.smithi033.stdout:1972: setattr oid 6 current snap is 177 2024-09-21T09:57:34.561 INFO:tasks.rados.rados.0.smithi033.stdout:1969: finishing copy_from racing read to smithi03385152-3 2024-09-21T09:57:34.561 INFO:tasks.rados.rados.0.smithi033.stdout:1970: finishing write tid 1 to smithi03385152-45 2024-09-21T09:57:34.561 INFO:tasks.rados.rados.0.smithi033.stdout:1973: read oid 50 snap -1 2024-09-21T09:57:34.561 INFO:tasks.rados.rados.0.smithi033.stdout:1973: expect deleted 2024-09-21T09:57:34.561 INFO:tasks.rados.rados.0.smithi033.stdout:1974: snap_remove snap 168 2024-09-21T09:57:34.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:57:34.695+0000 7fc38e955640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T09:57:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[98546]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:57:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[98546]: osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334] boot 2024-09-21T09:57:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[98546]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:57:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:57:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[98546]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:57:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[96568]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:57:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[96568]: osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334] boot 2024-09-21T09:57:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[96568]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:57:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:57:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:34 smithi033 ceph-mon[96568]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:57:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:34 smithi105 ceph-mon[86649]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:57:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:34 smithi105 ceph-mon[86649]: osd.3 [v2:172.21.15.33:6826/610865334,v1:172.21.15.33:6827/610865334] boot 2024-09-21T09:57:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:34 smithi105 ceph-mon[86649]: osdmap e426: 8 total, 8 up, 8 in 2024-09-21T09:57:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:34 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-21T09:57:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:34 smithi105 ceph-mon[86649]: osdmap e427: 8 total, 8 up, 8 in 2024-09-21T09:57:35.559 INFO:tasks.rados.rados.0.smithi033.stdout:1974: done (5 left) 2024-09-21T09:57:35.560 INFO:tasks.rados.rados.0.smithi033.stdout:1975: read oid 4 snap -1 2024-09-21T09:57:35.560 INFO:tasks.rados.rados.0.smithi033.stdout:1975: expect (ObjNum 612 snap 163 seq_num 612) 2024-09-21T09:57:35.560 INFO:tasks.rados.rados.0.smithi033.stdout:1976: read oid 38 snap -1 2024-09-21T09:57:35.560 INFO:tasks.rados.rados.0.smithi033.stdout:1976: expect (ObjNum 404 snap 110 seq_num 404) 2024-09-21T09:57:35.560 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 177 2024-09-21T09:57:35.560 INFO:tasks.rados.rados.0.smithi033.stdout:1977: seq_num 646 ranges {1867776=425984,2293760=655360,2949120=147456} 2024-09-21T09:57:35.573 INFO:tasks.rados.rados.0.smithi033.stdout:1977: writing smithi03385152-5 from 1867776 to 2293760 tid 1 2024-09-21T09:57:35.577 INFO:tasks.rados.rados.0.smithi033.stdout:1977: writing smithi03385152-5 from 2293760 to 2949120 tid 2 2024-09-21T09:57:35.579 INFO:tasks.rados.rados.0.smithi033.stdout:1977: writing smithi03385152-5 from 2949120 to 3096576 tid 3 2024-09-21T09:57:35.579 INFO:tasks.rados.rados.0.smithi033.stdout:1978: copy_from oid 43 from oid 19 current snap is 177 2024-09-21T09:57:35.579 INFO:tasks.rados.rados.0.smithi033.stdout:1969: finishing copy_from to smithi03385152-3 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 879 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1043 (ObjNum 587 snap 158 seq_num 587) dirty exists 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:1970: finishing write tid 2 to smithi03385152-45 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:1970: finishing write tid 3 to smithi03385152-45 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:1970: finishing write tid 4 to smithi03385152-45 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:1970: finishing write tid 5 to smithi03385152-45 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1024 (ObjNum 645 snap 177 seq_num 645) dirty exists 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:1970: left oid 45 (ObjNum 645 snap 177 seq_num 645) 2024-09-21T09:57:35.580 INFO:tasks.rados.rados.0.smithi033.stdout:1971: expect (ObjNum 609 snap 162 seq_num 609) 2024-09-21T09:57:35.726 INFO:tasks.rados.rados.0.smithi033.stdout:1976: expect (ObjNum 404 snap 110 seq_num 404) 2024-09-21T09:57:35.778 INFO:tasks.rados.rados.0.smithi033.stdout:1977: finishing write tid 1 to smithi03385152-5 2024-09-21T09:57:35.778 INFO:tasks.rados.rados.0.smithi033.stdout:1977: finishing write tid 2 to smithi03385152-5 2024-09-21T09:57:35.778 INFO:tasks.rados.rados.0.smithi033.stdout:1977: finishing write tid 3 to smithi03385152-5 2024-09-21T09:57:35.778 INFO:tasks.rados.rados.0.smithi033.stdout:1977: finishing write tid 4 to smithi03385152-5 2024-09-21T09:57:35.778 INFO:tasks.rados.rados.0.smithi033.stdout:1975: expect (ObjNum 612 snap 163 seq_num 612) 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:1977: finishing write tid 5 to smithi03385152-5 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 1006 (ObjNum 646 snap 177 seq_num 646) dirty exists 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:1977: left oid 5 (ObjNum 646 snap 177 seq_num 646) 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:1978: finishing copy_from racing read to smithi03385152-43 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:1978: finishing copy_from to smithi03385152-43 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1002 (ObjNum 600 snap 161 seq_num 600) dirty exists 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:1969: done (8 left) 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:1970: done (7 left) 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:1971: done (6 left) 2024-09-21T09:57:35.898 INFO:tasks.rados.rados.0.smithi033.stdout:1972: done (5 left) 2024-09-21T09:57:35.899 INFO:tasks.rados.rados.0.smithi033.stdout:1973: done (4 left) 2024-09-21T09:57:35.899 INFO:tasks.rados.rados.0.smithi033.stdout:1975: done (3 left) 2024-09-21T09:57:35.899 INFO:tasks.rados.rados.0.smithi033.stdout:1976: done (2 left) 2024-09-21T09:57:35.899 INFO:tasks.rados.rados.0.smithi033.stdout:1977: done (1 left) 2024-09-21T09:57:35.899 INFO:tasks.rados.rados.0.smithi033.stdout:1978: done (0 left) 2024-09-21T09:57:35.899 INFO:tasks.rados.rados.0.smithi033.stdout:1979: read oid 35 snap -1 2024-09-21T09:57:35.899 INFO:tasks.rados.rados.0.smithi033.stdout:1979: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:35.899 INFO:tasks.rados.rados.0.smithi033.stdout:1980: snap_create 2024-09-21T09:57:35.907 INFO:tasks.rados.rados.0.smithi033.stdout:1979: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: pgmap v269: 121 pgs: 3 peering, 32 active+undersized, 6 active+undersized+degraded, 80 active+clean; 203 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 20/345 objects degraded (5.797%) 2024-09-21T09:57:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[98546]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: pgmap v269: 121 pgs: 3 peering, 32 active+undersized, 6 active+undersized+degraded, 80 active+clean; 203 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 20/345 objects degraded (5.797%) 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T09:57:35.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:35 smithi033 ceph-mon[96568]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:57:36.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:35.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:36.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:35.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: pgmap v269: 121 pgs: 3 peering, 32 active+undersized, 6 active+undersized+degraded, 80 active+clean; 203 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 20/345 objects degraded (5.797%) 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:57:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: Upgrade: unsafe to stop osd(s) at this time (13 PGs are or would become offline) 2024-09-21T09:57:36.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:35 smithi105 ceph-mon[86649]: osdmap e428: 8 total, 8 up, 8 in 2024-09-21T09:57:36.835 INFO:tasks.rados.rados.0.smithi033.stdout:1979: done (1 left) 2024-09-21T09:57:36.835 INFO:tasks.rados.rados.0.smithi033.stdout:1980: done (0 left) 2024-09-21T09:57:36.835 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 178 2024-09-21T09:57:36.835 INFO:tasks.rados.rados.0.smithi033.stdout:1981: seq_num 647 ranges {1974272=442368,2416640=589824,3006464=212992} 2024-09-21T09:57:36.848 INFO:tasks.rados.rados.0.smithi033.stdout:1981: writing smithi03385152-38 from 1974272 to 2416640 tid 1 2024-09-21T09:57:36.851 INFO:tasks.rados.rados.0.smithi033.stdout:1981: writing smithi03385152-38 from 2416640 to 3006464 tid 2 2024-09-21T09:57:36.853 INFO:tasks.rados.rados.0.smithi033.stdout:1981: writing smithi03385152-38 from 3006464 to 3219456 tid 3 2024-09-21T09:57:36.853 INFO:tasks.rados.rados.0.smithi033.stdout:1982: delete oid 14 current snap is 178 2024-09-21T09:57:36.855 INFO:tasks.rados.rados.0.smithi033.stdout:1981: finishing write tid 1 to smithi03385152-38 2024-09-21T09:57:36.857 INFO:tasks.rados.rados.0.smithi033.stdout:1981: finishing write tid 2 to smithi03385152-38 2024-09-21T09:57:36.857 INFO:tasks.rados.rados.0.smithi033.stdout:1981: finishing write tid 3 to smithi03385152-38 2024-09-21T09:57:36.857 INFO:tasks.rados.rados.0.smithi033.stdout:1981: finishing write tid 4 to smithi03385152-38 2024-09-21T09:57:36.858 INFO:tasks.rados.rados.0.smithi033.stdout:1981: finishing write tid 5 to smithi03385152-38 2024-09-21T09:57:36.858 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 749 (ObjNum 647 snap 178 seq_num 647) dirty exists 2024-09-21T09:57:36.859 INFO:tasks.rados.rados.0.smithi033.stdout:1981: left oid 38 (ObjNum 647 snap 178 seq_num 647) 2024-09-21T09:57:36.860 INFO:tasks.rados.rados.0.smithi033.stdout:1981: done (1 left) 2024-09-21T09:57:36.860 INFO:tasks.rados.rados.0.smithi033.stdout:1982: done (0 left) 2024-09-21T09:57:36.860 INFO:tasks.rados.rados.0.smithi033.stdout:1983: rollback oid 19 current snap is 178 2024-09-21T09:57:36.860 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 19 to 177 2024-09-21T09:57:36.860 INFO:tasks.rados.rados.0.smithi033.stdout:1984: read oid 30 snap 174 2024-09-21T09:57:36.860 INFO:tasks.rados.rados.0.smithi033.stdout:1984: expect (ObjNum 624 snap 171 seq_num 624) 2024-09-21T09:57:36.860 INFO:tasks.rados.rados.0.smithi033.stdout:1985: rmattr oid 46 current snap is 178 2024-09-21T09:57:36.861 INFO:tasks.rados.rados.0.smithi033.stdout:1985: done (2 left) 2024-09-21T09:57:36.861 INFO:tasks.rados.rados.0.smithi033.stdout:1986: rmattr oid 6 current snap is 178 2024-09-21T09:57:36.861 INFO:tasks.rados.rados.0.smithi033.stdout:1987: copy_from oid 16 from oid 28 current snap is 178 2024-09-21T09:57:36.861 INFO:tasks.rados.rados.0.smithi033.stdout:1988: read oid 29 snap -1 2024-09-21T09:57:36.861 INFO:tasks.rados.rados.0.smithi033.stdout:1988: expect (ObjNum 529 snap 143 seq_num 529) 2024-09-21T09:57:36.861 INFO:tasks.rados.rados.0.smithi033.stdout:1989: snap_create 2024-09-21T09:57:36.863 INFO:tasks.rados.rados.0.smithi033.stdout:1983: finishing rollback tid 0 to smithi03385152-19 2024-09-21T09:57:36.864 INFO:tasks.rados.rados.0.smithi033.stdout:1983: finishing rollback tid 1 to smithi03385152-19 2024-09-21T09:57:36.864 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1044 (ObjNum 587 snap 158 seq_num 587) dirty exists 2024-09-21T09:57:36.864 INFO:tasks.rados.rados.0.smithi033.stdout:1983: finishing rollback tid 2 to smithi03385152-19 2024-09-21T09:57:36.864 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1005 (ObjNum 600 snap 161 seq_num 600) dirty exists 2024-09-21T09:57:36.917 INFO:tasks.rados.rados.0.smithi033.stdout:1984: expect (ObjNum 624 snap 171 seq_num 624) 2024-09-21T09:57:37.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:36 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:37.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:36 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:37.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:36 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:37.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:36.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:37.856 INFO:tasks.rados.rados.0.smithi033.stdout:1987: finishing copy_from racing read to smithi03385152-16 2024-09-21T09:57:37.856 INFO:tasks.rados.rados.0.smithi033.stdout:1987: finishing copy_from to smithi03385152-16 2024-09-21T09:57:37.856 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 879 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:57:37.856 INFO:tasks.rados.rados.0.smithi033.stdout:1988: expect (ObjNum 529 snap 143 seq_num 529) 2024-09-21T09:57:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:37 smithi033 ceph-mon[98546]: pgmap v272: 121 pgs: 9 peering, 15 active+undersized, 1 active+undersized+degraded, 96 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s; 2/297 objects degraded (0.673%); 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:57:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:37 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 2/297 objects degraded (0.673%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:57:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:37 smithi033 ceph-mon[98546]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:57:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:37 smithi033 ceph-mon[96568]: pgmap v272: 121 pgs: 9 peering, 15 active+undersized, 1 active+undersized+degraded, 96 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s; 2/297 objects degraded (0.673%); 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:57:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:37 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 2/297 objects degraded (0.673%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:57:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:37 smithi033 ceph-mon[96568]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:57:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:37 smithi105 ceph-mon[86649]: pgmap v272: 121 pgs: 9 peering, 15 active+undersized, 1 active+undersized+degraded, 96 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s; 2/297 objects degraded (0.673%); 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:57:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:37 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 2/297 objects degraded (0.673%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:57:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:37 smithi105 ceph-mon[86649]: osdmap e429: 8 total, 8 up, 8 in 2024-09-21T09:57:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:1983: done (5 left) 2024-09-21T09:57:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:1984: done (4 left) 2024-09-21T09:57:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:1986: done (3 left) 2024-09-21T09:57:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:1987: done (2 left) 2024-09-21T09:57:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:1988: done (1 left) 2024-09-21T09:57:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:1989: done (0 left) 2024-09-21T09:57:38.444 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 179 2024-09-21T09:57:38.445 INFO:tasks.rados.rados.0.smithi033.stdout:1990: seq_num 649 ranges {2646016=475136,3121152=761856,3883008=40960} 2024-09-21T09:57:38.463 INFO:tasks.rados.rados.0.smithi033.stdout:1990: writing smithi03385152-49 from 2646016 to 3121152 tid 1 2024-09-21T09:57:38.464 INFO:tasks.rados.rados.0.smithi033.stdout:1990: writing smithi03385152-49 from 3121152 to 3883008 tid 2 2024-09-21T09:57:38.465 INFO:tasks.rados.rados.0.smithi033.stdout:1990: writing smithi03385152-49 from 3883008 to 3923968 tid 3 2024-09-21T09:57:38.466 INFO:tasks.rados.rados.0.smithi033.stdout:1991: delete oid 40 current snap is 179 2024-09-21T09:57:38.466 INFO:tasks.rados.rados.0.smithi033.stdout:1990: finishing write tid 1 to smithi03385152-49 2024-09-21T09:57:38.468 INFO:tasks.rados.rados.0.smithi033.stdout:1991: done (1 left) 2024-09-21T09:57:38.468 INFO:tasks.rados.rados.0.smithi033.stdout:1992: read oid 6 snap 174 2024-09-21T09:57:38.468 INFO:tasks.rados.rados.0.smithi033.stdout:1992: expect (ObjNum 581 snap 156 seq_num 581) 2024-09-21T09:57:38.468 INFO:tasks.rados.rados.0.smithi033.stdout:1993: rollback oid 3 current snap is 179 2024-09-21T09:57:38.468 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 3 to 173 2024-09-21T09:57:38.469 INFO:tasks.rados.rados.0.smithi033.stdout:1994: read oid 42 snap -1 2024-09-21T09:57:38.469 INFO:tasks.rados.rados.0.smithi033.stdout:1994: expect (ObjNum 613 snap 163 seq_num 613) 2024-09-21T09:57:38.469 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 179 2024-09-21T09:57:38.469 INFO:tasks.rados.rados.0.smithi033.stdout:1995: seq_num 651 ranges {0=507904,507904=532480,1040384=270336} 2024-09-21T09:57:38.470 INFO:tasks.rados.rados.0.smithi033.stdout:1995: writing smithi03385152-41 from 0 to 507904 tid 1 2024-09-21T09:57:38.474 INFO:tasks.rados.rados.0.smithi033.stdout:1995: writing smithi03385152-41 from 507904 to 1040384 tid 2 2024-09-21T09:57:38.476 INFO:tasks.rados.rados.0.smithi033.stdout:1995: writing smithi03385152-41 from 1040384 to 1310720 tid 3 2024-09-21T09:57:38.476 INFO:tasks.rados.rados.0.smithi033.stdout:1996: read oid 28 snap -1 2024-09-21T09:57:38.476 INFO:tasks.rados.rados.0.smithi033.stdout:1996: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:57:38.476 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 179 2024-09-21T09:57:38.476 INFO:tasks.rados.rados.0.smithi033.stdout:1990: finishing write tid 2 to smithi03385152-49 2024-09-21T09:57:38.477 INFO:tasks.rados.rados.0.smithi033.stdout:1990: finishing write tid 3 to smithi03385152-49 2024-09-21T09:57:38.477 INFO:tasks.rados.rados.0.smithi033.stdout:1997: seq_num 652 ranges {876544=524288,1400832=409600,1810432=393216} 2024-09-21T09:57:38.483 INFO:tasks.rados.rados.0.smithi033.stdout:1997: writing smithi03385152-13 from 876544 to 1400832 tid 1 2024-09-21T09:57:38.486 INFO:tasks.rados.rados.0.smithi033.stdout:1997: writing smithi03385152-13 from 1400832 to 1810432 tid 2 2024-09-21T09:57:38.489 INFO:tasks.rados.rados.0.smithi033.stdout:1997: writing smithi03385152-13 from 1810432 to 2203648 tid 3 2024-09-21T09:57:38.489 INFO:tasks.rados.rados.0.smithi033.stdout:1998: snap_remove snap 175 2024-09-21T09:57:38.842 INFO:tasks.rados.rados.0.smithi033.stdout:1998: done (7 left) 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1999: rollback oid 33 current snap is 179 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 177 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1990: finishing write tid 4 to smithi03385152-49 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1990: finishing write tid 5 to smithi03385152-49 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1093 (ObjNum 649 snap 179 seq_num 649) dirty exists 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1990: left oid 49 (ObjNum 649 snap 179 seq_num 649) 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1993: finishing rollback tid 0 to smithi03385152-3 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1993: finishing rollback tid 1 to smithi03385152-3 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1993: finishing rollback tid 2 to smithi03385152-3 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1036 (ObjNum 450 snap 120 seq_num 450) dirty exists 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1995: finishing write tid 1 to smithi03385152-41 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1995: finishing write tid 2 to smithi03385152-41 2024-09-21T09:57:38.843 INFO:tasks.rados.rados.0.smithi033.stdout:1990: done (7 left) 2024-09-21T09:57:38.844 INFO:tasks.rados.rados.0.smithi033.stdout:1993: done (6 left) 2024-09-21T09:57:38.844 INFO:tasks.rados.rados.0.smithi033.stdout:2000: read oid 11 snap 176 2024-09-21T09:57:38.844 INFO:tasks.rados.rados.0.smithi033.stdout:2000: expect (ObjNum 626 snap 171 seq_num 626) 2024-09-21T09:57:38.844 INFO:tasks.rados.rados.0.smithi033.stdout:2001: setattr oid 29 current snap is 179 2024-09-21T09:57:38.844 INFO:tasks.rados.rados.0.smithi033.stdout:1992: expect (ObjNum 581 snap 156 seq_num 581) 2024-09-21T09:57:38.968 INFO:tasks.rados.rados.0.smithi033.stdout:1994: expect (ObjNum 613 snap 163 seq_num 613) 2024-09-21T09:57:39.102 INFO:tasks.rados.rados.0.smithi033.stdout:1995: finishing write tid 3 to smithi03385152-41 2024-09-21T09:57:39.102 INFO:tasks.rados.rados.0.smithi033.stdout:1996: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:57:39.161 INFO:tasks.rados.rados.0.smithi033.stdout:1995: finishing write tid 4 to smithi03385152-41 2024-09-21T09:57:39.161 INFO:tasks.rados.rados.0.smithi033.stdout:1995: finishing write tid 5 to smithi03385152-41 2024-09-21T09:57:39.161 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 754 (ObjNum 651 snap 179 seq_num 651) dirty exists 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:1995: left oid 41 (ObjNum 651 snap 179 seq_num 651) 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:1997: finishing write tid 1 to smithi03385152-13 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:1997: finishing write tid 2 to smithi03385152-13 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:1997: finishing write tid 3 to smithi03385152-13 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:1997: finishing write tid 4 to smithi03385152-13 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:1997: finishing write tid 5 to smithi03385152-13 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 1279 (ObjNum 652 snap 179 seq_num 652) dirty exists 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:1997: left oid 13 (ObjNum 652 snap 179 seq_num 652) 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:1999: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1036 (ObjNum 65 snap 0 seq_num 1718580339) dirty dne 2024-09-21T09:57:39.162 INFO:tasks.rados.rados.0.smithi033.stdout:2000: expect (ObjNum 626 snap 171 seq_num 626) 2024-09-21T09:57:39.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:38 smithi033 ceph-mon[96568]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:57:39.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:38 smithi033 ceph-mon[98546]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:57:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:38 smithi105 ceph-mon[86649]: osdmap e430: 8 total, 8 up, 8 in 2024-09-21T09:57:39.338 INFO:tasks.rados.rados.0.smithi033.stdout:1992: done (7 left) 2024-09-21T09:57:39.338 INFO:tasks.rados.rados.0.smithi033.stdout:1994: done (6 left) 2024-09-21T09:57:39.339 INFO:tasks.rados.rados.0.smithi033.stdout:1995: done (5 left) 2024-09-21T09:57:39.339 INFO:tasks.rados.rados.0.smithi033.stdout:1996: done (4 left) 2024-09-21T09:57:39.339 INFO:tasks.rados.rados.0.smithi033.stdout:1997: done (3 left) 2024-09-21T09:57:39.339 INFO:tasks.rados.rados.0.smithi033.stdout:1999: done (2 left) 2024-09-21T09:57:39.339 INFO:tasks.rados.rados.0.smithi033.stdout:2000: done (1 left) 2024-09-21T09:57:39.339 INFO:tasks.rados.rados.0.smithi033.stdout:2002: delete oid 19 current snap is 179 2024-09-21T09:57:39.340 INFO:tasks.rados.rados.0.smithi033.stdout:2002: done (1 left) 2024-09-21T09:57:39.340 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 179 2024-09-21T09:57:39.340 INFO:tasks.rados.rados.0.smithi033.stdout:2003: seq_num 654 ranges {0=557056,557056=630784,1187840=172032} 2024-09-21T09:57:39.343 INFO:tasks.rados.rados.0.smithi033.stdout:2003: writing smithi03385152-44 from 0 to 557056 tid 1 2024-09-21T09:57:39.347 INFO:tasks.rados.rados.0.smithi033.stdout:2003: writing smithi03385152-44 from 557056 to 1187840 tid 2 2024-09-21T09:57:39.348 INFO:tasks.rados.rados.0.smithi033.stdout:2003: writing smithi03385152-44 from 1187840 to 1359872 tid 3 2024-09-21T09:57:39.348 INFO:tasks.rados.rados.0.smithi033.stdout:2004: rollback oid 37 current snap is 179 2024-09-21T09:57:39.348 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 173 2024-09-21T09:57:39.349 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1283 (ObjNum 529 snap 143 seq_num 529) dirty exists 2024-09-21T09:57:39.349 INFO:tasks.rados.rados.0.smithi033.stdout:2003: finishing write tid 1 to smithi03385152-44 2024-09-21T09:57:39.349 INFO:tasks.rados.rados.0.smithi033.stdout:2001: done (2 left) 2024-09-21T09:57:39.349 INFO:tasks.rados.rados.0.smithi033.stdout:2005: read oid 14 snap -1 2024-09-21T09:57:39.349 INFO:tasks.rados.rados.0.smithi033.stdout:2005: expect deleted 2024-09-21T09:57:39.349 INFO:tasks.rados.rados.0.smithi033.stdout:2006: setattr oid 43 current snap is 179 2024-09-21T09:57:39.350 INFO:tasks.rados.rados.0.smithi033.stdout:2005: done (3 left) 2024-09-21T09:57:39.350 INFO:tasks.rados.rados.0.smithi033.stdout:2007: copy_from oid 45 from oid 35 current snap is 179 2024-09-21T09:57:39.350 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 179 2024-09-21T09:57:39.350 INFO:tasks.rados.rados.0.smithi033.stdout:2008: seq_num 655 ranges {10330112=573440,10903552=516096,11419648=286720} 2024-09-21T09:57:39.409 INFO:tasks.rados.rados.0.smithi033.stdout:2008: writing smithi03385152-21 from 10330112 to 10903552 tid 1 2024-09-21T09:57:39.412 INFO:tasks.rados.rados.0.smithi033.stdout:2008: writing smithi03385152-21 from 10903552 to 11419648 tid 2 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2008: writing smithi03385152-21 from 11419648 to 11706368 tid 3 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2009: rollback oid 10 current snap is 179 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 177 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2003: finishing write tid 2 to smithi03385152-44 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2003: finishing write tid 3 to smithi03385152-44 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2003: finishing write tid 4 to smithi03385152-44 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2004: finishing rollback tid 0 to smithi03385152-37 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2010: delete oid 29 current snap is 179 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2003: finishing write tid 5 to smithi03385152-44 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1025 (ObjNum 654 snap 179 seq_num 654) dirty exists 2024-09-21T09:57:39.415 INFO:tasks.rados.rados.0.smithi033.stdout:2003: left oid 44 (ObjNum 654 snap 179 seq_num 654) 2024-09-21T09:57:39.416 INFO:tasks.rados.rados.0.smithi033.stdout:2004: finishing rollback tid 1 to smithi03385152-37 2024-09-21T09:57:39.416 INFO:tasks.rados.rados.0.smithi033.stdout:2004: finishing rollback tid 2 to smithi03385152-37 2024-09-21T09:57:39.416 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1005 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:57:39.416 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1006 (ObjNum 600 snap 161 seq_num 600) dirty exists 2024-09-21T09:57:39.416 INFO:tasks.rados.rados.0.smithi033.stdout:2007: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:57:39.416 INFO:tasks.rados.rados.0.smithi033.stdout:2007: finishing copy_from to smithi03385152-45 2024-09-21T09:57:39.416 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 169 (ObjNum 574 snap 155 seq_num 574) dirty exists 2024-09-21T09:57:39.416 INFO:tasks.rados.rados.0.smithi033.stdout:2008: finishing write tid 1 to smithi03385152-21 2024-09-21T09:57:39.417 INFO:tasks.rados.rados.0.smithi033.stdout:2008: finishing write tid 2 to smithi03385152-21 2024-09-21T09:57:39.417 INFO:tasks.rados.rados.0.smithi033.stdout:2008: finishing write tid 3 to smithi03385152-21 2024-09-21T09:57:39.418 INFO:tasks.rados.rados.0.smithi033.stdout:2003: done (6 left) 2024-09-21T09:57:39.418 INFO:tasks.rados.rados.0.smithi033.stdout:2004: done (5 left) 2024-09-21T09:57:39.418 INFO:tasks.rados.rados.0.smithi033.stdout:2006: done (4 left) 2024-09-21T09:57:39.418 INFO:tasks.rados.rados.0.smithi033.stdout:2007: done (3 left) 2024-09-21T09:57:39.418 INFO:tasks.rados.rados.0.smithi033.stdout:2010: done (2 left) 2024-09-21T09:57:39.418 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 179 2024-09-21T09:57:39.418 INFO:tasks.rados.rados.0.smithi033.stdout:2009: finishing rollback tid 0 to smithi03385152-10 2024-09-21T09:57:39.418 INFO:tasks.rados.rados.0.smithi033.stdout:2011: seq_num 657 ranges {0=606208,606208=688128,1294336=114688} 2024-09-21T09:57:39.421 INFO:tasks.rados.rados.0.smithi033.stdout:2011: writing smithi03385152-9 from 0 to 606208 tid 1 2024-09-21T09:57:39.425 INFO:tasks.rados.rados.0.smithi033.stdout:2011: writing smithi03385152-9 from 606208 to 1294336 tid 2 2024-09-21T09:57:39.426 INFO:tasks.rados.rados.0.smithi033.stdout:2011: writing smithi03385152-9 from 1294336 to 1409024 tid 3 2024-09-21T09:57:39.426 INFO:tasks.rados.rados.0.smithi033.stdout:2012: rollback oid 25 current snap is 179 2024-09-21T09:57:39.426 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 25 to 167 2024-09-21T09:57:39.426 INFO:tasks.rados.rados.0.smithi033.stdout:2008: finishing write tid 4 to smithi03385152-21 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:2009: finishing rollback tid 1 to smithi03385152-10 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:2009: finishing rollback tid 2 to smithi03385152-10 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1009 (ObjNum 508 snap 135 seq_num 508) dirty exists 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:2008: finishing write tid 5 to smithi03385152-21 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 899 (ObjNum 655 snap 179 seq_num 655) dirty exists 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:2008: left oid 21 (ObjNum 655 snap 179 seq_num 655) 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:2011: finishing write tid 1 to smithi03385152-9 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:2008: done (3 left) 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:2009: done (2 left) 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 179 2024-09-21T09:57:39.427 INFO:tasks.rados.rados.0.smithi033.stdout:2013: seq_num 658 ranges {3006464=622592,3629056=565248,4194304=237568} 2024-09-21T09:57:39.446 INFO:tasks.rados.rados.0.smithi033.stdout:2013: writing smithi03385152-11 from 3006464 to 3629056 tid 1 2024-09-21T09:57:39.449 INFO:tasks.rados.rados.0.smithi033.stdout:2013: writing smithi03385152-11 from 3629056 to 4194304 tid 2 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2013: writing smithi03385152-11 from 4194304 to 4431872 tid 3 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2014: rollback oid 44 current snap is 179 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 44 to 174 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2011: finishing write tid 2 to smithi03385152-9 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2012: finishing rollback tid 0 to smithi03385152-25 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2015: snap_create 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2011: finishing write tid 3 to smithi03385152-9 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2011: finishing write tid 4 to smithi03385152-9 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2012: finishing rollback tid 1 to smithi03385152-25 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:2012: finishing rollback tid 2 to smithi03385152-25 2024-09-21T09:57:39.451 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 901 (ObjNum 535 snap 143 seq_num 535) dirty exists 2024-09-21T09:57:39.452 INFO:tasks.rados.rados.0.smithi033.stdout:2011: finishing write tid 5 to smithi03385152-9 2024-09-21T09:57:39.452 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 758 (ObjNum 657 snap 179 seq_num 657) dirty exists 2024-09-21T09:57:39.452 INFO:tasks.rados.rados.0.smithi033.stdout:2011: left oid 9 (ObjNum 657 snap 179 seq_num 657) 2024-09-21T09:57:39.452 INFO:tasks.rados.rados.0.smithi033.stdout:2013: finishing write tid 1 to smithi03385152-11 2024-09-21T09:57:39.452 INFO:tasks.rados.rados.0.smithi033.stdout:2013: finishing write tid 2 to smithi03385152-11 2024-09-21T09:57:39.452 INFO:tasks.rados.rados.0.smithi033.stdout:2013: finishing write tid 3 to smithi03385152-11 2024-09-21T09:57:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:2013: finishing write tid 4 to smithi03385152-11 2024-09-21T09:57:39.453 INFO:tasks.rados.rados.0.smithi033.stdout:2014: finishing rollback tid 0 to smithi03385152-44 2024-09-21T09:57:39.454 INFO:tasks.rados.rados.0.smithi033.stdout:2014: finishing rollback tid 1 to smithi03385152-44 2024-09-21T09:57:39.454 INFO:tasks.rados.rados.0.smithi033.stdout:2013: finishing write tid 5 to smithi03385152-11 2024-09-21T09:57:39.454 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1010 (ObjNum 658 snap 179 seq_num 658) dirty exists 2024-09-21T09:57:39.454 INFO:tasks.rados.rados.0.smithi033.stdout:2013: left oid 11 (ObjNum 658 snap 179 seq_num 658) 2024-09-21T09:57:39.454 INFO:tasks.rados.rados.0.smithi033.stdout:2014: finishing rollback tid 2 to smithi03385152-44 2024-09-21T09:57:39.454 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1028 (ObjNum 608 snap 162 seq_num 608) dirty exists 2024-09-21T09:57:39.850 INFO:tasks.rados.rados.0.smithi033.stdout:2011: done (4 left) 2024-09-21T09:57:39.850 INFO:tasks.rados.rados.0.smithi033.stdout:2012: done (3 left) 2024-09-21T09:57:39.850 INFO:tasks.rados.rados.0.smithi033.stdout:2013: done (2 left) 2024-09-21T09:57:39.850 INFO:tasks.rados.rados.0.smithi033.stdout:2014: done (1 left) 2024-09-21T09:57:39.850 INFO:tasks.rados.rados.0.smithi033.stdout:2015: done (0 left) 2024-09-21T09:57:39.851 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 180 2024-09-21T09:57:39.851 INFO:tasks.rados.rados.0.smithi033.stdout:2016: seq_num 659 ranges {0=647168,647168=450560,1097728=352256} 2024-09-21T09:57:39.854 INFO:tasks.rados.rados.0.smithi033.stdout:2016: writing smithi03385152-50 from 0 to 647168 tid 1 2024-09-21T09:57:39.856 INFO:tasks.rados.rados.0.smithi033.stdout:2016: writing smithi03385152-50 from 647168 to 1097728 tid 2 2024-09-21T09:57:39.859 INFO:tasks.rados.rados.0.smithi033.stdout:2016: writing smithi03385152-50 from 1097728 to 1449984 tid 3 2024-09-21T09:57:39.859 INFO:tasks.rados.rados.0.smithi033.stdout:2017: read oid 28 snap 177 2024-09-21T09:57:39.859 INFO:tasks.rados.rados.0.smithi033.stdout:2017: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:57:39.859 INFO:tasks.rados.rados.0.smithi033.stdout:2018: snap_remove snap 167 2024-09-21T09:57:40.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:39 smithi105 ceph-mon[86649]: pgmap v275: 121 pgs: 9 peering, 112 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:57:40.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:39 smithi105 ceph-mon[86649]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/297 objects degraded (0.673%), 1 pg degraded) 2024-09-21T09:57:40.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:39 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T09:57:40.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:39 smithi105 ceph-mon[86649]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:57:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:39 smithi033 ceph-mon[98546]: pgmap v275: 121 pgs: 9 peering, 112 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:57:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:39 smithi033 ceph-mon[98546]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/297 objects degraded (0.673%), 1 pg degraded) 2024-09-21T09:57:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:39 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T09:57:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:39 smithi033 ceph-mon[98546]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:57:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:39 smithi033 ceph-mon[96568]: pgmap v275: 121 pgs: 9 peering, 112 active+clean; 161 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 1.0 MiB/s wr, 12 op/s; 2.8 MiB/s, 1 objects/s recovering 2024-09-21T09:57:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:39 smithi033 ceph-mon[96568]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/297 objects degraded (0.673%), 1 pg degraded) 2024-09-21T09:57:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:39 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T09:57:40.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:39 smithi033 ceph-mon[96568]: osdmap e431: 8 total, 8 up, 8 in 2024-09-21T09:57:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:40.863 INFO:tasks.rados.rados.0.smithi033.stdout:2018: done (2 left) 2024-09-21T09:57:40.863 INFO:tasks.rados.rados.0.smithi033.stdout:2019: read oid 36 snap -1 2024-09-21T09:57:40.863 INFO:tasks.rados.rados.0.smithi033.stdout:2019: expect (ObjNum 592 snap 159 seq_num 592) 2024-09-21T09:57:40.863 INFO:tasks.rados.rados.0.smithi033.stdout:2020: read oid 4 snap 177 2024-09-21T09:57:40.863 INFO:tasks.rados.rados.0.smithi033.stdout:2020: expect (ObjNum 612 snap 163 seq_num 612) 2024-09-21T09:57:40.863 INFO:tasks.rados.rados.0.smithi033.stdout:2021: rollback oid 12 current snap is 180 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 12 to 179 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2022: read oid 41 snap 173 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2022: expect deleted 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2023: snap_create 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2016: finishing write tid 1 to smithi03385152-50 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2016: finishing write tid 2 to smithi03385152-50 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2016: finishing write tid 3 to smithi03385152-50 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2016: finishing write tid 4 to smithi03385152-50 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2016: finishing write tid 5 to smithi03385152-50 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 905 (ObjNum 659 snap 180 seq_num 659) dirty exists 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2016: left oid 50 (ObjNum 659 snap 180 seq_num 659) 2024-09-21T09:57:40.864 INFO:tasks.rados.rados.0.smithi033.stdout:2017: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:57:40.931 INFO:tasks.rados.rados.0.smithi033.stdout:2021: finishing rollback tid 1 to smithi03385152-12 2024-09-21T09:57:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 0 (ObjNum 129 snap 0 seq_num 1497640136) dirty dne 2024-09-21T09:57:40.932 INFO:tasks.rados.rados.0.smithi033.stdout:2020: expect (ObjNum 612 snap 163 seq_num 612) 2024-09-21T09:57:41.050 INFO:tasks.rados.rados.0.smithi033.stdout:2019: expect (ObjNum 592 snap 159 seq_num 592) 2024-09-21T09:57:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:40 smithi033 ceph-mon[98546]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:57:41.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:40 smithi033 ceph-mon[96568]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:57:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:40 smithi105 ceph-mon[86649]: osdmap e432: 8 total, 8 up, 8 in 2024-09-21T09:57:41.859 INFO:tasks.rados.rados.0.smithi033.stdout:2016: done (6 left) 2024-09-21T09:57:41.859 INFO:tasks.rados.rados.0.smithi033.stdout:2017: done (5 left) 2024-09-21T09:57:41.859 INFO:tasks.rados.rados.0.smithi033.stdout:2019: done (4 left) 2024-09-21T09:57:41.859 INFO:tasks.rados.rados.0.smithi033.stdout:2020: done (3 left) 2024-09-21T09:57:41.860 INFO:tasks.rados.rados.0.smithi033.stdout:2021: done (2 left) 2024-09-21T09:57:41.860 INFO:tasks.rados.rados.0.smithi033.stdout:2022: done (1 left) 2024-09-21T09:57:41.860 INFO:tasks.rados.rados.0.smithi033.stdout:2023: done (0 left) 2024-09-21T09:57:41.860 INFO:tasks.rados.rados.0.smithi033.stdout:2024: snap_create 2024-09-21T09:57:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:41 smithi033 ceph-mon[98546]: pgmap v278: 121 pgs: 121 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 322 KiB/s, 0 objects/s recovering 2024-09-21T09:57:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:41 smithi033 ceph-mon[98546]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:57:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:41 smithi033 ceph-mon[96568]: pgmap v278: 121 pgs: 121 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 322 KiB/s, 0 objects/s recovering 2024-09-21T09:57:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:41 smithi033 ceph-mon[96568]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:57:41.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:57:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:57:41] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.51.0" 2024-09-21T09:57:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:41 smithi105 ceph-mon[86649]: pgmap v278: 121 pgs: 121 active+clean; 167 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 14 op/s; 322 KiB/s, 0 objects/s recovering 2024-09-21T09:57:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:41 smithi105 ceph-mon[86649]: osdmap e433: 8 total, 8 up, 8 in 2024-09-21T09:57:42.861 INFO:tasks.rados.rados.0.smithi033.stdout:2024: done (0 left) 2024-09-21T09:57:42.862 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 182 2024-09-21T09:57:42.862 INFO:tasks.rados.rados.0.smithi033.stdout:2025: seq_num 660 ranges {1449984=663552,2113536=737280,2850816=65536} 2024-09-21T09:57:42.873 INFO:tasks.rados.rados.0.smithi033.stdout:2025: writing smithi03385152-50 from 1449984 to 2113536 tid 1 2024-09-21T09:57:42.877 INFO:tasks.rados.rados.0.smithi033.stdout:2025: writing smithi03385152-50 from 2113536 to 2850816 tid 2 2024-09-21T09:57:42.878 INFO:tasks.rados.rados.0.smithi033.stdout:2025: writing smithi03385152-50 from 2850816 to 2916352 tid 3 2024-09-21T09:57:42.878 INFO:tasks.rados.rados.0.smithi033.stdout:2026: rmattr oid 14 current snap is 182 2024-09-21T09:57:42.878 INFO:tasks.rados.rados.0.smithi033.stdout:2026: done (1 left) 2024-09-21T09:57:42.878 INFO:tasks.rados.rados.0.smithi033.stdout:2027: read oid 8 snap -1 2024-09-21T09:57:42.879 INFO:tasks.rados.rados.0.smithi033.stdout:2027: expect (ObjNum 642 snap 176 seq_num 642) 2024-09-21T09:57:42.879 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 182 2024-09-21T09:57:42.879 INFO:tasks.rados.rados.0.smithi033.stdout:2028: seq_num 661 ranges {1318912=679936,1998848=622592,2621440=180224} 2024-09-21T09:57:42.890 INFO:tasks.rados.rados.0.smithi033.stdout:2028: writing smithi03385152-48 from 1318912 to 1998848 tid 1 2024-09-21T09:57:42.894 INFO:tasks.rados.rados.0.smithi033.stdout:2028: writing smithi03385152-48 from 1998848 to 2621440 tid 2 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2028: writing smithi03385152-48 from 2621440 to 2801664 tid 3 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2029: rollback oid 15 current snap is 182 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 15 to 179 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2025: finishing write tid 1 to smithi03385152-50 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2025: finishing write tid 2 to smithi03385152-50 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2025: finishing write tid 3 to smithi03385152-50 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2025: finishing write tid 4 to smithi03385152-50 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2030: read oid 45 snap -1 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2030: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 182 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2025: finishing write tid 5 to smithi03385152-50 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 909 (ObjNum 660 snap 182 seq_num 660) dirty exists 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2025: left oid 50 (ObjNum 660 snap 182 seq_num 660) 2024-09-21T09:57:42.896 INFO:tasks.rados.rados.0.smithi033.stdout:2027: expect (ObjNum 642 snap 176 seq_num 642) 2024-09-21T09:57:42.962 INFO:tasks.rados.rados.0.smithi033.stdout:2029: finishing rollback tid 0 to smithi03385152-15 2024-09-21T09:57:42.962 INFO:tasks.rados.rados.0.smithi033.stdout:2029: finishing rollback tid 1 to smithi03385152-15 2024-09-21T09:57:42.962 INFO:tasks.rados.rados.0.smithi033.stdout:2029: finishing rollback tid 2 to smithi03385152-15 2024-09-21T09:57:42.962 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 900 (ObjNum 630 snap 171 seq_num 630) dirty exists 2024-09-21T09:57:42.962 INFO:tasks.rados.rados.0.smithi033.stdout:2028: finishing write tid 1 to smithi03385152-48 2024-09-21T09:57:42.962 INFO:tasks.rados.rados.0.smithi033.stdout:2031: seq_num 662 ranges {1368064=696320,2064384=557056,2621440=245760} 2024-09-21T09:57:42.973 INFO:tasks.rados.rados.0.smithi033.stdout:2031: writing smithi03385152-27 from 1368064 to 2064384 tid 1 2024-09-21T09:57:42.976 INFO:tasks.rados.rados.0.smithi033.stdout:2031: writing smithi03385152-27 from 2064384 to 2621440 tid 2 2024-09-21T09:57:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:2031: writing smithi03385152-27 from 2621440 to 2867200 tid 3 2024-09-21T09:57:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:2025: done (5 left) 2024-09-21T09:57:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:2027: done (4 left) 2024-09-21T09:57:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:2029: done (3 left) 2024-09-21T09:57:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:2032: snap_create 2024-09-21T09:57:42.978 INFO:tasks.rados.rados.0.smithi033.stdout:2028: finishing write tid 2 to smithi03385152-48 2024-09-21T09:57:42.979 INFO:tasks.rados.rados.0.smithi033.stdout:2028: finishing write tid 3 to smithi03385152-48 2024-09-21T09:57:42.979 INFO:tasks.rados.rados.0.smithi033.stdout:2028: finishing write tid 4 to smithi03385152-48 2024-09-21T09:57:42.979 INFO:tasks.rados.rados.0.smithi033.stdout:2028: finishing write tid 5 to smithi03385152-48 2024-09-21T09:57:42.979 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 1059 (ObjNum 661 snap 182 seq_num 661) dirty exists 2024-09-21T09:57:42.979 INFO:tasks.rados.rados.0.smithi033.stdout:2028: left oid 48 (ObjNum 661 snap 182 seq_num 661) 2024-09-21T09:57:42.979 INFO:tasks.rados.rados.0.smithi033.stdout:2030: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:43.057 INFO:tasks.rados.rados.0.smithi033.stdout:2031: finishing write tid 1 to smithi03385152-27 2024-09-21T09:57:43.058 INFO:tasks.rados.rados.0.smithi033.stdout:2031: finishing write tid 2 to smithi03385152-27 2024-09-21T09:57:43.058 INFO:tasks.rados.rados.0.smithi033.stdout:2031: finishing write tid 3 to smithi03385152-27 2024-09-21T09:57:43.058 INFO:tasks.rados.rados.0.smithi033.stdout:2031: finishing write tid 4 to smithi03385152-27 2024-09-21T09:57:43.058 INFO:tasks.rados.rados.0.smithi033.stdout:2031: finishing write tid 5 to smithi03385152-27 2024-09-21T09:57:43.058 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1048 (ObjNum 662 snap 182 seq_num 662) dirty exists 2024-09-21T09:57:43.058 INFO:tasks.rados.rados.0.smithi033.stdout:2031: left oid 27 (ObjNum 662 snap 182 seq_num 662) 2024-09-21T09:57:43.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:42 smithi033 ceph-mon[98546]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:57:43.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:42 smithi033 ceph-mon[98546]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:57:43.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:42 smithi033 ceph-mon[96568]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:57:43.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:42 smithi033 ceph-mon[96568]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:57:43.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:42 smithi105 ceph-mon[86649]: osdmap e434: 8 total, 8 up, 8 in 2024-09-21T09:57:43.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:42 smithi105 ceph-mon[86649]: osdmap e435: 8 total, 8 up, 8 in 2024-09-21T09:57:43.472 INFO:tasks.rados.rados.0.smithi033.stdout:2028: done (3 left) 2024-09-21T09:57:43.473 INFO:tasks.rados.rados.0.smithi033.stdout:2030: done (2 left) 2024-09-21T09:57:43.473 INFO:tasks.rados.rados.0.smithi033.stdout:2031: done (1 left) 2024-09-21T09:57:43.473 INFO:tasks.rados.rados.0.smithi033.stdout:2032: done (0 left) 2024-09-21T09:57:43.473 INFO:tasks.rados.rados.0.smithi033.stdout:2033: snap_create 2024-09-21T09:57:44.148 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:43 smithi105 ceph-mon[86649]: pgmap v281: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.9 MiB/s wr, 37 op/s; 322 KiB/s, 0 objects/s recovering 2024-09-21T09:57:44.148 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:43 smithi105 ceph-mon[86649]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:57:44.189 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:43 smithi033 ceph-mon[98546]: pgmap v281: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.9 MiB/s wr, 37 op/s; 322 KiB/s, 0 objects/s recovering 2024-09-21T09:57:44.189 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:43 smithi033 ceph-mon[98546]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:57:44.189 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:43 smithi033 ceph-mon[96568]: pgmap v281: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 3.9 MiB/s wr, 37 op/s; 322 KiB/s, 0 objects/s recovering 2024-09-21T09:57:44.190 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:43 smithi033 ceph-mon[96568]: osdmap e436: 8 total, 8 up, 8 in 2024-09-21T09:57:44.477 INFO:tasks.rados.rados.0.smithi033.stdout:2033: done (0 left) 2024-09-21T09:57:44.477 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 184 2024-09-21T09:57:44.477 INFO:tasks.rados.rados.0.smithi033.stdout:2034: seq_num 663 ranges {4431872=712704,5144576=442368,5586944=360448} 2024-09-21T09:57:44.505 INFO:tasks.rados.rados.0.smithi033.stdout:2034: writing smithi03385152-11 from 4431872 to 5144576 tid 1 2024-09-21T09:57:44.507 INFO:tasks.rados.rados.0.smithi033.stdout:2034: writing smithi03385152-11 from 5144576 to 5586944 tid 2 2024-09-21T09:57:44.510 INFO:tasks.rados.rados.0.smithi033.stdout:2034: writing smithi03385152-11 from 5586944 to 5947392 tid 3 2024-09-21T09:57:44.510 INFO:tasks.rados.rados.0.smithi033.stdout:2035: delete oid 48 current snap is 184 2024-09-21T09:57:44.510 INFO:tasks.rados.rados.0.smithi033.stdout:2034: finishing write tid 1 to smithi03385152-11 2024-09-21T09:57:44.510 INFO:tasks.rados.rados.0.smithi033.stdout:2034: finishing write tid 2 to smithi03385152-11 2024-09-21T09:57:44.512 INFO:tasks.rados.rados.0.smithi033.stdout:2035: done (1 left) 2024-09-21T09:57:44.512 INFO:tasks.rados.rados.0.smithi033.stdout:2036: snap_create 2024-09-21T09:57:44.512 INFO:tasks.rados.rados.0.smithi033.stdout:2034: finishing write tid 3 to smithi03385152-11 2024-09-21T09:57:44.513 INFO:tasks.rados.rados.0.smithi033.stdout:2034: finishing write tid 4 to smithi03385152-11 2024-09-21T09:57:44.514 INFO:tasks.rados.rados.0.smithi033.stdout:2034: finishing write tid 5 to smithi03385152-11 2024-09-21T09:57:44.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1014 (ObjNum 663 snap 184 seq_num 663) dirty exists 2024-09-21T09:57:44.514 INFO:tasks.rados.rados.0.smithi033.stdout:2034: left oid 11 (ObjNum 663 snap 184 seq_num 663) 2024-09-21T09:57:44.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:44.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:57:44.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:44.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.3\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.3\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.33\", device_class=\"ssd\", hostname=\"smithi033\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.33\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:45.481 INFO:tasks.rados.rados.0.smithi033.stdout:2034: done (1 left) 2024-09-21T09:57:45.481 INFO:tasks.rados.rados.0.smithi033.stdout:2036: done (0 left) 2024-09-21T09:57:45.481 INFO:tasks.rados.rados.0.smithi033.stdout:2037: read oid 1 snap -1 2024-09-21T09:57:45.481 INFO:tasks.rados.rados.0.smithi033.stdout:2037: expect deleted 2024-09-21T09:57:45.481 INFO:tasks.rados.rados.0.smithi033.stdout:2038: read oid 17 snap -1 2024-09-21T09:57:45.481 INFO:tasks.rados.rados.0.smithi033.stdout:2038: expect (ObjNum 541 snap 145 seq_num 541) 2024-09-21T09:57:45.481 INFO:tasks.rados.rados.0.smithi033.stdout:2039: read oid 42 snap 174 2024-09-21T09:57:45.481 INFO:tasks.rados.rados.0.smithi033.stdout:2039: expect (ObjNum 613 snap 163 seq_num 613) 2024-09-21T09:57:45.482 INFO:tasks.rados.rados.0.smithi033.stdout:2040: rmattr oid 39 current snap is 185 2024-09-21T09:57:45.482 INFO:tasks.rados.rados.0.smithi033.stdout:2040: done (3 left) 2024-09-21T09:57:45.482 INFO:tasks.rados.rados.0.smithi033.stdout:2041: read oid 12 snap -1 2024-09-21T09:57:45.482 INFO:tasks.rados.rados.0.smithi033.stdout:2041: expect deleted 2024-09-21T09:57:45.482 INFO:tasks.rados.rados.0.smithi033.stdout:2042: setattr oid 48 current snap is 185 2024-09-21T09:57:45.482 INFO:tasks.rados.rados.0.smithi033.stdout:2037: done (4 left) 2024-09-21T09:57:45.482 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 185 2024-09-21T09:57:45.482 INFO:tasks.rados.rados.0.smithi033.stdout:2043: seq_num 665 ranges {2203648=745472,2949120=606208,3555328=196608} 2024-09-21T09:57:45.498 INFO:tasks.rados.rados.0.smithi033.stdout:2043: writing smithi03385152-13 from 2203648 to 2949120 tid 1 2024-09-21T09:57:45.502 INFO:tasks.rados.rados.0.smithi033.stdout:2043: writing smithi03385152-13 from 2949120 to 3555328 tid 2 2024-09-21T09:57:45.504 INFO:tasks.rados.rados.0.smithi033.stdout:2043: writing smithi03385152-13 from 3555328 to 3751936 tid 3 2024-09-21T09:57:45.504 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 185 2024-09-21T09:57:45.504 INFO:tasks.rados.rados.0.smithi033.stdout:2044: seq_num 666 ranges {0=761856,761856=491520,1253376=311296} 2024-09-21T09:57:45.508 INFO:tasks.rados.rados.0.smithi033.stdout:2044: writing smithi03385152-14 from 0 to 761856 tid 1 2024-09-21T09:57:45.511 INFO:tasks.rados.rados.0.smithi033.stdout:2044: writing smithi03385152-14 from 761856 to 1253376 tid 2 2024-09-21T09:57:45.514 INFO:tasks.rados.rados.0.smithi033.stdout:2044: writing smithi03385152-14 from 1253376 to 1564672 tid 3 2024-09-21T09:57:45.514 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 185 2024-09-21T09:57:45.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 1062 (ObjNum 0 snap 0 seq_num 304) dirty exists 2024-09-21T09:57:45.514 INFO:tasks.rados.rados.0.smithi033.stdout:2039: expect (ObjNum 613 snap 163 seq_num 613) 2024-09-21T09:57:45.651 INFO:tasks.rados.rados.0.smithi033.stdout:2043: finishing write tid 1 to smithi03385152-13 2024-09-21T09:57:45.651 INFO:tasks.rados.rados.0.smithi033.stdout:2043: finishing write tid 2 to smithi03385152-13 2024-09-21T09:57:45.651 INFO:tasks.rados.rados.0.smithi033.stdout:2043: finishing write tid 3 to smithi03385152-13 2024-09-21T09:57:45.651 INFO:tasks.rados.rados.0.smithi033.stdout:2043: finishing write tid 4 to smithi03385152-13 2024-09-21T09:57:45.651 INFO:tasks.rados.rados.0.smithi033.stdout:2043: finishing write tid 5 to smithi03385152-13 2024-09-21T09:57:45.651 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 1299 (ObjNum 665 snap 185 seq_num 665) dirty exists 2024-09-21T09:57:45.652 INFO:tasks.rados.rados.0.smithi033.stdout:2043: left oid 13 (ObjNum 665 snap 185 seq_num 665) 2024-09-21T09:57:45.652 INFO:tasks.rados.rados.0.smithi033.stdout:2044: finishing write tid 1 to smithi03385152-14 2024-09-21T09:57:45.652 INFO:tasks.rados.rados.0.smithi033.stdout:2045: seq_num 667 ranges {0=778240,778240=778240,1556480=24576} 2024-09-21T09:57:45.655 INFO:tasks.rados.rados.0.smithi033.stdout:2045: writing smithi03385152-46 from 0 to 778240 tid 1 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2045: writing smithi03385152-46 from 778240 to 1556480 tid 2 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2045: writing smithi03385152-46 from 1556480 to 1581056 tid 3 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2039: done (6 left) 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2041: done (5 left) 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2042: done (4 left) 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2043: done (3 left) 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2046: delete oid 7 current snap is 185 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2044: finishing write tid 2 to smithi03385152-14 2024-09-21T09:57:45.660 INFO:tasks.rados.rados.0.smithi033.stdout:2044: finishing write tid 3 to smithi03385152-14 2024-09-21T09:57:45.661 INFO:tasks.rados.rados.0.smithi033.stdout:2044: finishing write tid 4 to smithi03385152-14 2024-09-21T09:57:45.661 INFO:tasks.rados.rados.0.smithi033.stdout:2044: finishing write tid 5 to smithi03385152-14 2024-09-21T09:57:45.661 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1066 (ObjNum 666 snap 185 seq_num 666) dirty exists 2024-09-21T09:57:45.661 INFO:tasks.rados.rados.0.smithi033.stdout:2044: left oid 14 (ObjNum 666 snap 185 seq_num 666) 2024-09-21T09:57:45.661 INFO:tasks.rados.rados.0.smithi033.stdout:2038: expect (ObjNum 541 snap 145 seq_num 541) 2024-09-21T09:57:45.741 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:45 smithi105 ceph-mon[86649]: pgmap v284: 121 pgs: 121 active+clean; 156 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:57:45.742 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:45 smithi105 ceph-mon[86649]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:57:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:45 smithi033 ceph-mon[98546]: pgmap v284: 121 pgs: 121 active+clean; 156 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:57:45.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:45 smithi033 ceph-mon[98546]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:57:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:45 smithi033 ceph-mon[96568]: pgmap v284: 121 pgs: 121 active+clean; 156 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 2.8 MiB/s wr, 24 op/s 2024-09-21T09:57:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:45 smithi033 ceph-mon[96568]: osdmap e437: 8 total, 8 up, 8 in 2024-09-21T09:57:46.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:45.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:46.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:45.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:46.397 INFO:tasks.rados.rados.0.smithi033.stdout:2045: finishing write tid 1 to smithi03385152-46 2024-09-21T09:57:46.397 INFO:tasks.rados.rados.0.smithi033.stdout:2045: finishing write tid 2 to smithi03385152-46 2024-09-21T09:57:46.397 INFO:tasks.rados.rados.0.smithi033.stdout:2045: finishing write tid 3 to smithi03385152-46 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2045: finishing write tid 4 to smithi03385152-46 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2045: finishing write tid 5 to smithi03385152-46 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1070 (ObjNum 667 snap 185 seq_num 667) dirty exists 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2045: left oid 46 (ObjNum 667 snap 185 seq_num 667) 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2038: done (3 left) 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2044: done (2 left) 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2045: done (1 left) 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2046: done (0 left) 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2047: copy_from oid 17 from oid 37 current snap is 185 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 24 current snap is 185 2024-09-21T09:57:46.398 INFO:tasks.rados.rados.0.smithi033.stdout:2048: seq_num 669 ranges {0=409600,409600=548864,958464=655360} 2024-09-21T09:57:46.399 INFO:tasks.rados.rados.0.smithi033.stdout:2048: writing smithi03385152-24 from 0 to 409600 tid 1 2024-09-21T09:57:46.402 INFO:tasks.rados.rados.0.smithi033.stdout:2048: writing smithi03385152-24 from 409600 to 958464 tid 2 2024-09-21T09:57:46.407 INFO:tasks.rados.rados.0.smithi033.stdout:2048: writing smithi03385152-24 from 958464 to 1613824 tid 3 2024-09-21T09:57:46.407 INFO:tasks.rados.rados.0.smithi033.stdout:2049: delete oid 1 current snap is 185 2024-09-21T09:57:46.407 INFO:tasks.rados.rados.0.smithi033.stdout:2048: finishing write tid 1 to smithi03385152-24 2024-09-21T09:57:46.407 INFO:tasks.rados.rados.0.smithi033.stdout:2048: finishing write tid 2 to smithi03385152-24 2024-09-21T09:57:46.409 INFO:tasks.rados.rados.0.smithi033.stdout:2049: done (2 left) 2024-09-21T09:57:46.409 INFO:tasks.rados.rados.0.smithi033.stdout:2050: rmattr oid 2 current snap is 185 2024-09-21T09:57:46.409 INFO:tasks.rados.rados.0.smithi033.stdout:2051: snap_create 2024-09-21T09:57:46.410 INFO:tasks.rados.rados.0.smithi033.stdout:2048: finishing write tid 3 to smithi03385152-24 2024-09-21T09:57:46.410 INFO:tasks.rados.rados.0.smithi033.stdout:2048: finishing write tid 4 to smithi03385152-24 2024-09-21T09:57:46.412 INFO:tasks.rados.rados.0.smithi033.stdout:2048: finishing write tid 5 to smithi03385152-24 2024-09-21T09:57:46.412 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 1013 (ObjNum 669 snap 185 seq_num 669) dirty exists 2024-09-21T09:57:46.412 INFO:tasks.rados.rados.0.smithi033.stdout:2048: left oid 24 (ObjNum 669 snap 185 seq_num 669) 2024-09-21T09:57:46.412 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 910 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-21T09:57:46.427 INFO:tasks.rados.rados.0.smithi033.stdout:2047: finishing copy_from racing read to smithi03385152-17 2024-09-21T09:57:46.433 INFO:tasks.rados.rados.0.smithi033.stdout:2047: finishing copy_from to smithi03385152-17 2024-09-21T09:57:46.433 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 1005 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:57:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:46 smithi033 ceph-mon[96568]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:57:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:46.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:46 smithi033 ceph-mon[98546]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:57:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:46.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:46 smithi105 ceph-mon[86649]: osdmap e438: 8 total, 8 up, 8 in 2024-09-21T09:57:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:57:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:46.836 INFO:tasks.rados.rados.0.smithi033.stdout:2047: done (3 left) 2024-09-21T09:57:46.836 INFO:tasks.rados.rados.0.smithi033.stdout:2048: done (2 left) 2024-09-21T09:57:46.836 INFO:tasks.rados.rados.0.smithi033.stdout:2050: done (1 left) 2024-09-21T09:57:46.837 INFO:tasks.rados.rados.0.smithi033.stdout:2051: done (0 left) 2024-09-21T09:57:46.837 INFO:tasks.rados.rados.0.smithi033.stdout:2052: rollback oid 36 current snap is 186 2024-09-21T09:57:46.837 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 180 2024-09-21T09:57:46.837 INFO:tasks.rados.rados.0.smithi033.stdout:2053: delete oid 18 current snap is 186 2024-09-21T09:57:46.841 INFO:tasks.rados.rados.0.smithi033.stdout:2053: done (1 left) 2024-09-21T09:57:46.842 INFO:tasks.rados.rados.0.smithi033.stdout:2054: read oid 49 snap -1 2024-09-21T09:57:46.842 INFO:tasks.rados.rados.0.smithi033.stdout:2054: expect (ObjNum 649 snap 179 seq_num 649) 2024-09-21T09:57:46.842 INFO:tasks.rados.rados.0.smithi033.stdout:2055: copy_from oid 32 from oid 1 current snap is 186 2024-09-21T09:57:46.842 INFO:tasks.rados.rados.0.smithi033.stdout:2056: rollback oid 43 current snap is 186 2024-09-21T09:57:46.842 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 43 to 174 2024-09-21T09:57:46.842 INFO:tasks.rados.rados.0.smithi033.stdout:2057: rollback oid 41 current snap is 186 2024-09-21T09:57:46.842 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 180 2024-09-21T09:57:46.842 INFO:tasks.rados.rados.0.smithi033.stdout:2058: snap_create 2024-09-21T09:57:46.845 INFO:tasks.rados.rados.0.smithi033.stdout:2052: finishing rollback tid 0 to smithi03385152-36 2024-09-21T09:57:46.847 INFO:tasks.rados.rados.0.smithi033.stdout:2056: finishing rollback tid 0 to smithi03385152-43 2024-09-21T09:57:46.847 INFO:tasks.rados.rados.0.smithi033.stdout:2055: finishing copy_from to smithi03385152-32 2024-09-21T09:57:46.847 INFO:tasks.rados.rados.0.smithi033.stdout:2055: got expected ENOENT (src dne) 2024-09-21T09:57:46.847 INFO:tasks.rados.rados.0.smithi033.stdout:2056: finishing rollback tid 1 to smithi03385152-43 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:2052: finishing rollback tid 1 to smithi03385152-36 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:2056: finishing rollback tid 2 to smithi03385152-43 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1017 (ObjNum 633 snap 171 seq_num 633) dirty exists 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:2052: finishing rollback tid 2 to smithi03385152-36 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1305 (ObjNum 592 snap 159 seq_num 592) dirty exists 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:2055: finishing copy_from racing read to smithi03385152-32 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:2057: finishing rollback tid 0 to smithi03385152-41 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:2057: finishing rollback tid 1 to smithi03385152-41 2024-09-21T09:57:46.848 INFO:tasks.rados.rados.0.smithi033.stdout:2057: finishing rollback tid 2 to smithi03385152-41 2024-09-21T09:57:46.849 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 761 (ObjNum 651 snap 179 seq_num 651) dirty exists 2024-09-21T09:57:46.861 INFO:tasks.rados.rados.0.smithi033.stdout:2054: expect (ObjNum 649 snap 179 seq_num 649) 2024-09-21T09:57:47.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:46.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:47.596 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:57:47.840 INFO:tasks.rados.rados.0.smithi033.stdout:2052: done (5 left) 2024-09-21T09:57:47.840 INFO:tasks.rados.rados.0.smithi033.stdout:2054: done (4 left) 2024-09-21T09:57:47.840 INFO:tasks.rados.rados.0.smithi033.stdout:2055: done (3 left) 2024-09-21T09:57:47.840 INFO:tasks.rados.rados.0.smithi033.stdout:2056: done (2 left) 2024-09-21T09:57:47.840 INFO:tasks.rados.rados.0.smithi033.stdout:2057: done (1 left) 2024-09-21T09:57:47.840 INFO:tasks.rados.rados.0.smithi033.stdout:2058: done (0 left) 2024-09-21T09:57:47.840 INFO:tasks.rados.rados.0.smithi033.stdout:2059: read oid 42 snap 174 2024-09-21T09:57:47.840 INFO:tasks.rados.rados.0.smithi033.stdout:2059: expect (ObjNum 613 snap 163 seq_num 613) 2024-09-21T09:57:47.841 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 187 2024-09-21T09:57:47.841 INFO:tasks.rados.rados.0.smithi033.stdout:2060: seq_num 672 ranges {0=458752,458752=647168,1105920=466944,1572864=90112} 2024-09-21T09:57:47.843 INFO:tasks.rados.rados.0.smithi033.stdout:2060: writing smithi03385152-12 from 0 to 458752 tid 1 2024-09-21T09:57:47.847 INFO:tasks.rados.rados.0.smithi033.stdout:2060: writing smithi03385152-12 from 458752 to 1105920 tid 2 2024-09-21T09:57:47.850 INFO:tasks.rados.rados.0.smithi033.stdout:2060: writing smithi03385152-12 from 1105920 to 1572864 tid 3 2024-09-21T09:57:47.850 INFO:tasks.rados.rados.0.smithi033.stdout:2060: writing smithi03385152-12 from 1572864 to 1662976 tid 4 2024-09-21T09:57:47.851 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 187 2024-09-21T09:57:47.851 INFO:tasks.rados.rados.0.smithi033.stdout:2061: seq_num 673 ranges {9404416=475136,9879552=532480,10412032=671744} 2024-09-21T09:57:47.904 INFO:tasks.rados.rados.0.smithi033.stdout:2061: writing smithi03385152-22 from 9404416 to 9879552 tid 1 2024-09-21T09:57:47.907 INFO:tasks.rados.rados.0.smithi033.stdout:2061: writing smithi03385152-22 from 9879552 to 10412032 tid 2 2024-09-21T09:57:47.911 INFO:tasks.rados.rados.0.smithi033.stdout:2061: writing smithi03385152-22 from 10412032 to 11083776 tid 3 2024-09-21T09:57:47.911 INFO:tasks.rados.rados.0.smithi033.stdout:2062: rollback oid 35 current snap is 187 2024-09-21T09:57:47.911 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 35 to 177 2024-09-21T09:57:47.911 INFO:tasks.rados.rados.0.smithi033.stdout:2059: expect (ObjNum 613 snap 163 seq_num 613) 2024-09-21T09:57:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:47 smithi033 ceph-mon[98546]: pgmap v287: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.9 MiB/s wr, 19 op/s 2024-09-21T09:57:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:47 smithi033 ceph-mon[98546]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:57:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:47 smithi033 ceph-mon[96568]: pgmap v287: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.9 MiB/s wr, 19 op/s 2024-09-21T09:57:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:47 smithi033 ceph-mon[96568]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2060: finishing write tid 1 to smithi03385152-12 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2060: finishing write tid 2 to smithi03385152-12 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2060: finishing write tid 3 to smithi03385152-12 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2060: finishing write tid 4 to smithi03385152-12 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2060: finishing write tid 5 to smithi03385152-12 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2060: finishing write tid 6 to smithi03385152-12 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1098 (ObjNum 672 snap 187 seq_num 672) dirty exists 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2060: left oid 12 (ObjNum 672 snap 187 seq_num 672) 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2061: finishing write tid 1 to smithi03385152-22 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2061: finishing write tid 2 to smithi03385152-22 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2061: finishing write tid 3 to smithi03385152-22 2024-09-21T09:57:48.064 INFO:tasks.rados.rados.0.smithi033.stdout:2061: finishing write tid 4 to smithi03385152-22 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2059: done (3 left) 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2060: done (2 left) 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2063: rollback oid 50 current snap is 187 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 50 to 178 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2062: finishing rollback tid 0 to smithi03385152-35 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2062: finishing rollback tid 1 to smithi03385152-35 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2062: finishing rollback tid 2 to smithi03385152-35 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 1309 (ObjNum 574 snap 155 seq_num 574) dirty exists 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2061: finishing write tid 5 to smithi03385152-22 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1075 (ObjNum 673 snap 187 seq_num 673) dirty exists 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2061: left oid 22 (ObjNum 673 snap 187 seq_num 673) 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2061: done (2 left) 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2062: done (1 left) 2024-09-21T09:57:48.065 INFO:tasks.rados.rados.0.smithi033.stdout:2064: read oid 35 snap -1 2024-09-21T09:57:48.066 INFO:tasks.rados.rados.0.smithi033.stdout:2064: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:48.066 INFO:tasks.rados.rados.0.smithi033.stdout:2065: rmattr oid 36 current snap is 187 2024-09-21T09:57:48.066 INFO:tasks.rados.rados.0.smithi033.stdout:2066: setattr oid 43 current snap is 187 2024-09-21T09:57:48.066 INFO:tasks.rados.rados.0.smithi033.stdout:2067: delete oid 3 current snap is 187 2024-09-21T09:57:48.067 INFO:tasks.rados.rados.0.smithi033.stdout:2063: finishing rollback tid 0 to smithi03385152-50 2024-09-21T09:57:48.067 INFO:tasks.rados.rados.0.smithi033.stdout:2063: finishing rollback tid 1 to smithi03385152-50 2024-09-21T09:57:48.067 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 912 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:57:48.069 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1018 (ObjNum 633 snap 171 seq_num 633) dirty exists 2024-09-21T09:57:48.074 INFO:tasks.rados.rados.0.smithi033.stdout:2064: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:48.152 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1311 (ObjNum 592 snap 159 seq_num 592) dirty exists 2024-09-21T09:57:48.153 INFO:tasks.rados.rados.0.smithi033.stdout:2063: done (4 left) 2024-09-21T09:57:48.153 INFO:tasks.rados.rados.0.smithi033.stdout:2064: done (3 left) 2024-09-21T09:57:48.153 INFO:tasks.rados.rados.0.smithi033.stdout:2065: done (2 left) 2024-09-21T09:57:48.153 INFO:tasks.rados.rados.0.smithi033.stdout:2066: done (1 left) 2024-09-21T09:57:48.153 INFO:tasks.rados.rados.0.smithi033.stdout:2067: done (0 left) 2024-09-21T09:57:48.153 INFO:tasks.rados.rados.0.smithi033.stdout:2068: copy_from oid 23 from oid 37 current snap is 187 2024-09-21T09:57:48.153 INFO:tasks.rados.rados.0.smithi033.stdout:2069: delete oid 17 current snap is 187 2024-09-21T09:57:48.155 INFO:tasks.rados.rados.0.smithi033.stdout:2069: done (1 left) 2024-09-21T09:57:48.155 INFO:tasks.rados.rados.0.smithi033.stdout:2070: read oid 29 snap -1 2024-09-21T09:57:48.155 INFO:tasks.rados.rados.0.smithi033.stdout:2070: expect deleted 2024-09-21T09:57:48.155 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 187 2024-09-21T09:57:48.155 INFO:tasks.rados.rados.0.smithi033.stdout:2071: seq_num 676 ranges {0=524288,524288=589824,1114112=450560,1564672=163840} 2024-09-21T09:57:48.158 INFO:tasks.rados.rados.0.smithi033.stdout:2071: writing smithi03385152-20 from 0 to 524288 tid 1 2024-09-21T09:57:48.162 INFO:tasks.rados.rados.0.smithi033.stdout:2071: writing smithi03385152-20 from 524288 to 1114112 tid 2 2024-09-21T09:57:48.165 INFO:tasks.rados.rados.0.smithi033.stdout:2071: writing smithi03385152-20 from 1114112 to 1564672 tid 3 2024-09-21T09:57:48.166 INFO:tasks.rados.rados.0.smithi033.stdout:2071: writing smithi03385152-20 from 1564672 to 1728512 tid 4 2024-09-21T09:57:48.166 INFO:tasks.rados.rados.0.smithi033.stdout:2072: read oid 43 snap -1 2024-09-21T09:57:48.166 INFO:tasks.rados.rados.0.smithi033.stdout:2072: expect (ObjNum 633 snap 171 seq_num 633) 2024-09-21T09:57:48.167 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 187 2024-09-21T09:57:48.167 INFO:tasks.rados.rados.0.smithi033.stdout:2073: seq_num 677 ranges {0=540672,540672=516096,1056768=655360,1712128=32768} 2024-09-21T09:57:48.170 INFO:tasks.rados.rados.0.smithi033.stdout:2073: writing smithi03385152-34 from 0 to 540672 tid 1 2024-09-21T09:57:48.173 INFO:tasks.rados.rados.0.smithi033.stdout:2073: writing smithi03385152-34 from 540672 to 1056768 tid 2 2024-09-21T09:57:48.177 INFO:tasks.rados.rados.0.smithi033.stdout:2073: writing smithi03385152-34 from 1056768 to 1712128 tid 3 2024-09-21T09:57:48.178 INFO:tasks.rados.rados.0.smithi033.stdout:2073: writing smithi03385152-34 from 1712128 to 1744896 tid 4 2024-09-21T09:57:48.178 INFO:tasks.rados.rados.0.smithi033.stdout:2074: rollback oid 44 current snap is 187 2024-09-21T09:57:48.178 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 44 to 183 2024-09-21T09:57:48.178 INFO:tasks.rados.rados.0.smithi033.stdout:2071: finishing write tid 1 to smithi03385152-20 2024-09-21T09:57:48.178 INFO:tasks.rados.rados.0.smithi033.stdout:2073: finishing write tid 1 to smithi03385152-34 2024-09-21T09:57:48.178 INFO:tasks.rados.rados.0.smithi033.stdout:2072: expect (ObjNum 633 snap 171 seq_num 633) 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (4m) 17s ago 9m 19.2M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (4m) 2m ago 8m 82.4M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (10m) 17s ago 10m 16.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (10m) 2m ago 10m 19.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (3m) 2m ago 14m 453M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (4m) 17s ago 15m 524M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (3m) 17s ago 15m 56.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (2m) 2m ago 14m 22.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (3m) 17s ago 14m 47.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:57:48.219 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (4m) 17s ago 8m 11.8M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (4m) 2m ago 8m 9915k - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (2m) 17s ago 13m 168M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (89s) 17s ago 13m 126M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (55s) 17s ago 12m 66.9M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 76dbf29565ba 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (21s) 17s ago 12m 11.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b85b61817088 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (12m) 2m ago 12m 167M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 02f11869fd82 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (11m) 2m ago 11m 193M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (11m) 2m ago 11m 126M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (10m) 2m ago 10m 140M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:57:48.220 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (3m) 2m ago 9m 44.0M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:57:48.247 INFO:tasks.rados.rados.0.smithi033.stdout:2073: finishing write tid 2 to smithi03385152-34 2024-09-21T09:57:48.247 INFO:tasks.rados.rados.0.smithi033.stdout:2070: done (5 left) 2024-09-21T09:57:48.247 INFO:tasks.rados.rados.0.smithi033.stdout:2072: done (4 left) 2024-09-21T09:57:48.247 INFO:tasks.rados.rados.0.smithi033.stdout:2075: delete oid 24 current snap is 187 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2068: finishing copy_from racing read to smithi03385152-23 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2074: finishing rollback tid 0 to smithi03385152-44 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2073: finishing write tid 3 to smithi03385152-34 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2073: finishing write tid 4 to smithi03385152-34 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2073: finishing write tid 5 to smithi03385152-34 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2073: finishing write tid 6 to smithi03385152-34 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1082 (ObjNum 677 snap 187 seq_num 677) dirty exists 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2073: left oid 34 (ObjNum 677 snap 187 seq_num 677) 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2074: finishing rollback tid 1 to smithi03385152-44 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2068: finishing copy_from to smithi03385152-23 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1005 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2074: finishing rollback tid 2 to smithi03385152-44 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1031 (ObjNum 608 snap 162 seq_num 608) dirty exists 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2071: finishing write tid 2 to smithi03385152-20 2024-09-21T09:57:48.248 INFO:tasks.rados.rados.0.smithi033.stdout:2071: finishing write tid 3 to smithi03385152-20 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2071: finishing write tid 4 to smithi03385152-20 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2071: finishing write tid 5 to smithi03385152-20 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2071: finishing write tid 6 to smithi03385152-20 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1009 (ObjNum 676 snap 187 seq_num 676) dirty exists 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2071: left oid 20 (ObjNum 676 snap 187 seq_num 676) 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2068: done (4 left) 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2071: done (3 left) 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2073: done (2 left) 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2074: done (1 left) 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2075: done (0 left) 2024-09-21T09:57:48.249 INFO:tasks.rados.rados.0.smithi033.stdout:2076: setattr oid 29 current snap is 187 2024-09-21T09:57:48.251 INFO:tasks.rados.rados.0.smithi033.stdout:2077: rollback oid 3 current snap is 187 2024-09-21T09:57:48.252 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 3 to 183 2024-09-21T09:57:48.252 INFO:tasks.rados.rados.0.smithi033.stdout:2078: rollback oid 20 current snap is 187 2024-09-21T09:57:48.252 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 181 2024-09-21T09:57:48.252 INFO:tasks.rados.rados.0.smithi033.stdout:2079: read oid 43 snap 177 2024-09-21T09:57:48.252 INFO:tasks.rados.rados.0.smithi033.stdout:2079: expect (ObjNum 600 snap 161 seq_num 600) 2024-09-21T09:57:48.252 INFO:tasks.rados.rados.0.smithi033.stdout:2080: delete oid 8 current snap is 187 2024-09-21T09:57:48.254 INFO:tasks.rados.rados.0.smithi033.stdout:2078: finishing rollback tid 0 to smithi03385152-20 2024-09-21T09:57:48.255 INFO:tasks.rados.rados.0.smithi033.stdout:2080: done (4 left) 2024-09-21T09:57:48.255 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 187 2024-09-21T09:57:48.255 INFO:tasks.rados.rados.0.smithi033.stdout:2081: seq_num 680 ranges {11706368=598016,12304384=573440,12877824=434176,13312000=196608} 2024-09-21T09:57:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:47 smithi105 ceph-mon[86649]: pgmap v287: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.9 MiB/s wr, 19 op/s 2024-09-21T09:57:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:47 smithi105 ceph-mon[86649]: osdmap e439: 8 total, 8 up, 8 in 2024-09-21T09:57:48.322 INFO:tasks.rados.rados.0.smithi033.stdout:2081: writing smithi03385152-21 from 11706368 to 12304384 tid 1 2024-09-21T09:57:48.325 INFO:tasks.rados.rados.0.smithi033.stdout:2081: writing smithi03385152-21 from 12304384 to 12877824 tid 2 2024-09-21T09:57:48.328 INFO:tasks.rados.rados.0.smithi033.stdout:2081: writing smithi03385152-21 from 12877824 to 13312000 tid 3 2024-09-21T09:57:48.330 INFO:tasks.rados.rados.0.smithi033.stdout:2081: writing smithi03385152-21 from 13312000 to 13508608 tid 4 2024-09-21T09:57:48.330 INFO:tasks.rados.rados.0.smithi033.stdout:2077: finishing rollback tid 1 to smithi03385152-3 2024-09-21T09:57:48.330 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1312 (ObjNum 49 snap 0 seq_num 1) dirty exists 2024-09-21T09:57:48.330 INFO:tasks.rados.rados.0.smithi033.stdout:2078: finishing rollback tid 1 to smithi03385152-20 2024-09-21T09:57:48.330 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1011 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:57:48.330 INFO:tasks.rados.rados.0.smithi033.stdout:2077: finishing rollback tid 2 to smithi03385152-3 2024-09-21T09:57:48.330 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1084 (ObjNum 450 snap 120 seq_num 450) dirty exists 2024-09-21T09:57:48.330 INFO:tasks.rados.rados.0.smithi033.stdout:2079: expect (ObjNum 600 snap 161 seq_num 600) 2024-09-21T09:57:48.342 INFO:tasks.rados.rados.0.smithi033.stdout:2081: finishing write tid 1 to smithi03385152-21 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2081: finishing write tid 2 to smithi03385152-21 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2081: finishing write tid 3 to smithi03385152-21 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2081: finishing write tid 4 to smithi03385152-21 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2081: finishing write tid 5 to smithi03385152-21 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2081: finishing write tid 6 to smithi03385152-21 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1016 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2081: left oid 21 (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2076: done (4 left) 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2077: done (3 left) 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2078: done (2 left) 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2079: done (1 left) 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2081: done (0 left) 2024-09-21T09:57:48.343 INFO:tasks.rados.rados.0.smithi033.stdout:2082: delete oid 8 current snap is 187 2024-09-21T09:57:48.344 INFO:tasks.rados.rados.0.smithi033.stdout:2082: done (0 left) 2024-09-21T09:57:48.344 INFO:tasks.rados.rados.0.smithi033.stdout:2083: snap_remove snap 181 2024-09-21T09:57:48.475 INFO:tasks.rados.rados.0.smithi033.stdout:2083: done (0 left) 2024-09-21T09:57:48.475 INFO:tasks.rados.rados.0.smithi033.stdout:2084: snap_remove snap 184 2024-09-21T09:57:48.595 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:57:48.595 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 4, 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 4 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:57:48.596 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 6, 2024-09-21T09:57:48.597 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 9 2024-09-21T09:57:48.597 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:57:48.597 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:57:48.845 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[96568]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:48.845 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[96568]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:57:48.846 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[96568]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:48.846 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[96568]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:57:48.897 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:57:48.897 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:57:48.897 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:57:48.897 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:57:48.897 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:57:48.897 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:57:48.897 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:57:48.897 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:57:48.898 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "9/20 daemons upgraded", 2024-09-21T09:57:48.898 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:57:48.898 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:57:48.898 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:57:49.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3149261121' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[98546]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[98546]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:57:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[98546]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[98546]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:57:49.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:48 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3149261121' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:48 smithi105 ceph-mon[86649]: from='client.34362 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:48 smithi105 ceph-mon[86649]: osdmap e440: 8 total, 8 up, 8 in 2024-09-21T09:57:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:48 smithi105 ceph-mon[86649]: from='client.34368 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:48 smithi105 ceph-mon[86649]: osdmap e441: 8 total, 8 up, 8 in 2024-09-21T09:57:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:48 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3149261121' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:57:49.476 INFO:tasks.rados.rados.0.smithi033.stdout:2084: done (0 left) 2024-09-21T09:57:49.476 INFO:tasks.rados.rados.0.smithi033.stdout:2085: rollback oid 34 current snap is 187 2024-09-21T09:57:49.476 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 185 2024-09-21T09:57:49.476 INFO:tasks.rados.rados.0.smithi033.stdout:2086: snap_remove snap 183 2024-09-21T09:57:50.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:49 smithi105 ceph-mon[86649]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:50.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:49 smithi105 ceph-mon[86649]: pgmap v290: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.5 MiB/s wr, 18 op/s 2024-09-21T09:57:50.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:49 smithi105 ceph-mon[86649]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:50.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:49 smithi105 ceph-mon[86649]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:57:50.186 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:49 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:57:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[96568]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[96568]: pgmap v290: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.5 MiB/s wr, 18 op/s 2024-09-21T09:57:50.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[96568]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:50.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[96568]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:57:50.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:57:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[98546]: from='client.44288 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[98546]: pgmap v290: 121 pgs: 121 active+clean; 166 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 2.5 MiB/s wr, 18 op/s 2024-09-21T09:57:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[98546]: from='client.34386 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:57:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[98546]: osdmap e442: 8 total, 8 up, 8 in 2024-09-21T09:57:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:49 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16} : dispatch 2024-09-21T09:57:50.442 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:50.481 INFO:tasks.rados.rados.0.smithi033.stdout:2086: done (1 left) 2024-09-21T09:57:50.481 INFO:tasks.rados.rados.0.smithi033.stdout:2087: copy_from oid 27 from oid 37 current snap is 187 2024-09-21T09:57:50.482 INFO:tasks.rados.rados.0.smithi033.stdout:2088: snap_remove snap 180 2024-09-21T09:57:51.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:50 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:57:51.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:50 smithi105 ceph-mon[86649]: Upgrade: osd.4 is safe to restart 2024-09-21T09:57:51.139 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:50 smithi105 ceph-mon[86649]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:57:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:50 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:57:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:50 smithi033 ceph-mon[96568]: Upgrade: osd.4 is safe to restart 2024-09-21T09:57:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:50 smithi033 ceph-mon[96568]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:57:51.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:50 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-21T09:57:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:50 smithi033 ceph-mon[98546]: Upgrade: osd.4 is safe to restart 2024-09-21T09:57:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:50 smithi033 ceph-mon[98546]: osdmap e443: 8 total, 8 up, 8 in 2024-09-21T09:57:51.479 INFO:tasks.rados.rados.0.smithi033.stdout:2088: done (2 left) 2024-09-21T09:57:51.479 INFO:tasks.rados.rados.0.smithi033.stdout:2089: delete oid 45 current snap is 187 2024-09-21T09:57:51.480 INFO:tasks.rados.rados.0.smithi033.stdout:2085: finishing rollback tid 0 to smithi03385152-34 2024-09-21T09:57:51.480 INFO:tasks.rados.rados.0.smithi033.stdout:2085: finishing rollback tid 1 to smithi03385152-34 2024-09-21T09:57:51.480 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1092 (ObjNum 33 snap 0 seq_num 4) dirty dne 2024-09-21T09:57:51.480 INFO:tasks.rados.rados.0.smithi033.stdout:2087: finishing copy_from racing read to smithi03385152-27 2024-09-21T09:57:51.480 INFO:tasks.rados.rados.0.smithi033.stdout:2087: finishing copy_from to smithi03385152-27 2024-09-21T09:57:51.480 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1005 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:57:51.488 INFO:tasks.rados.rados.0.smithi033.stdout:2085: done (2 left) 2024-09-21T09:57:51.489 INFO:tasks.rados.rados.0.smithi033.stdout:2087: done (1 left) 2024-09-21T09:57:51.489 INFO:tasks.rados.rados.0.smithi033.stdout:2089: done (0 left) 2024-09-21T09:57:51.489 INFO:tasks.rados.rados.0.smithi033.stdout:2090: read oid 47 snap -1 2024-09-21T09:57:51.489 INFO:tasks.rados.rados.0.smithi033.stdout:2090: expect (ObjNum 598 snap 161 seq_num 598) 2024-09-21T09:57:51.489 INFO:tasks.rados.rados.0.smithi033.stdout:2091: read oid 2 snap -1 2024-09-21T09:57:51.489 INFO:tasks.rados.rados.0.smithi033.stdout:2091: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:57:51.489 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 187 2024-09-21T09:57:51.489 INFO:tasks.rados.rados.0.smithi033.stdout:2092: seq_num 683 ranges {2678784=647168,3325952=630784,3956736=573440} 2024-09-21T09:57:51.507 INFO:tasks.rados.rados.0.smithi033.stdout:2092: writing smithi03385152-42 from 2678784 to 3325952 tid 1 2024-09-21T09:57:51.511 INFO:tasks.rados.rados.0.smithi033.stdout:2092: writing smithi03385152-42 from 3325952 to 3956736 tid 2 2024-09-21T09:57:51.514 INFO:tasks.rados.rados.0.smithi033.stdout:2092: writing smithi03385152-42 from 3956736 to 4530176 tid 3 2024-09-21T09:57:51.514 INFO:tasks.rados.rados.0.smithi033.stdout:2093: snap_create 2024-09-21T09:57:51.515 INFO:tasks.rados.rados.0.smithi033.stdout:2091: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:57:51.516 INFO:tasks.rados.rados.0.smithi033.stdout:2090: expect (ObjNum 598 snap 161 seq_num 598) 2024-09-21T09:57:51.655 INFO:tasks.rados.rados.0.smithi033.stdout:2092: finishing write tid 1 to smithi03385152-42 2024-09-21T09:57:51.655 INFO:tasks.rados.rados.0.smithi033.stdout:2092: finishing write tid 2 to smithi03385152-42 2024-09-21T09:57:51.655 INFO:tasks.rados.rados.0.smithi033.stdout:2092: finishing write tid 3 to smithi03385152-42 2024-09-21T09:57:51.655 INFO:tasks.rados.rados.0.smithi033.stdout:2092: finishing write tid 4 to smithi03385152-42 2024-09-21T09:57:51.655 INFO:tasks.rados.rados.0.smithi033.stdout:2092: finishing write tid 5 to smithi03385152-42 2024-09-21T09:57:51.655 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 765 (ObjNum 683 snap 187 seq_num 683) dirty exists 2024-09-21T09:57:51.655 INFO:tasks.rados.rados.0.smithi033.stdout:2092: left oid 42 (ObjNum 683 snap 187 seq_num 683) 2024-09-21T09:57:51.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:57:51] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.51.0" 2024-09-21T09:57:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[98546]: pgmap v293: 121 pgs: 121 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 394 KiB/s wr, 12 op/s 2024-09-21T09:57:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:57:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[98546]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:57:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[96568]: pgmap v293: 121 pgs: 121 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 394 KiB/s wr, 12 op/s 2024-09-21T09:57:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:57:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:51 smithi033 ceph-mon[96568]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:57:52.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:51 smithi105 ceph-mon[86649]: pgmap v293: 121 pgs: 121 active+clean; 169 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 394 KiB/s wr, 12 op/s 2024-09-21T09:57:52.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:57:52.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-21T09:57:52.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:57:52.246 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:51 smithi105 ceph-mon[86649]: osdmap e444: 8 total, 8 up, 8 in 2024-09-21T09:57:52.481 INFO:tasks.rados.rados.0.smithi033.stdout:2090: done (3 left) 2024-09-21T09:57:52.481 INFO:tasks.rados.rados.0.smithi033.stdout:2091: done (2 left) 2024-09-21T09:57:52.481 INFO:tasks.rados.rados.0.smithi033.stdout:2092: done (1 left) 2024-09-21T09:57:52.481 INFO:tasks.rados.rados.0.smithi033.stdout:2093: done (0 left) 2024-09-21T09:57:52.481 INFO:tasks.rados.rados.0.smithi033.stdout:2094: rmattr oid 27 current snap is 188 2024-09-21T09:57:52.481 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 188 2024-09-21T09:57:52.482 INFO:tasks.rados.rados.0.smithi033.stdout:2095: seq_num 684 ranges {958464=663552,1622016=507904,2129920=409600,2539520=286720} 2024-09-21T09:57:52.490 INFO:tasks.rados.rados.0.smithi033.stdout:2095: writing smithi03385152-15 from 958464 to 1622016 tid 1 2024-09-21T09:57:52.493 INFO:tasks.rados.rados.0.smithi033.stdout:2095: writing smithi03385152-15 from 1622016 to 2129920 tid 2 2024-09-21T09:57:52.496 INFO:tasks.rados.rados.0.smithi033.stdout:2095: writing smithi03385152-15 from 2129920 to 2539520 tid 3 2024-09-21T09:57:52.498 INFO:tasks.rados.rados.0.smithi033.stdout:2095: writing smithi03385152-15 from 2539520 to 2826240 tid 4 2024-09-21T09:57:52.498 INFO:tasks.rados.rados.0.smithi033.stdout:2096: rollback oid 28 current snap is 188 2024-09-21T09:57:52.498 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 187 2024-09-21T09:57:52.498 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1049 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:57:52.498 INFO:tasks.rados.rados.0.smithi033.stdout:2095: finishing write tid 1 to smithi03385152-15 2024-09-21T09:57:52.498 INFO:tasks.rados.rados.0.smithi033.stdout:2094: done (2 left) 2024-09-21T09:57:52.498 INFO:tasks.rados.rados.0.smithi033.stdout:2097: snap_remove snap 182 2024-09-21T09:57:53.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:52 smithi105 ceph-mon[86649]: Upgrade: Updating osd.4 2024-09-21T09:57:53.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:52 smithi105 ceph-mon[86649]: Deploying daemon osd.4 on smithi105 2024-09-21T09:57:53.141 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:52 smithi105 ceph-mon[86649]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:57:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:52 smithi033 ceph-mon[98546]: Upgrade: Updating osd.4 2024-09-21T09:57:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:52 smithi033 ceph-mon[98546]: Deploying daemon osd.4 on smithi105 2024-09-21T09:57:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:52 smithi033 ceph-mon[98546]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:57:53.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:52 smithi033 ceph-mon[96568]: Upgrade: Updating osd.4 2024-09-21T09:57:53.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:52 smithi033 ceph-mon[96568]: Deploying daemon osd.4 on smithi105 2024-09-21T09:57:53.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:52 smithi033 ceph-mon[96568]: osdmap e445: 8 total, 8 up, 8 in 2024-09-21T09:57:53.489 INFO:tasks.rados.rados.0.smithi033.stdout:2097: done (2 left) 2024-09-21T09:57:53.489 INFO:tasks.rados.rados.0.smithi033.stdout:2098: rmattr oid 29 current snap is 188 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2095: finishing write tid 2 to smithi03385152-15 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2095: finishing write tid 3 to smithi03385152-15 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2095: finishing write tid 4 to smithi03385152-15 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2095: finishing write tid 5 to smithi03385152-15 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2096: finishing rollback tid 0 to smithi03385152-28 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2096: finishing rollback tid 1 to smithi03385152-28 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2096: finishing rollback tid 2 to smithi03385152-28 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 882 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2095: finishing write tid 6 to smithi03385152-15 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 906 (ObjNum 684 snap 188 seq_num 684) dirty exists 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2095: left oid 15 (ObjNum 684 snap 188 seq_num 684) 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2095: done (2 left) 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2096: done (1 left) 2024-09-21T09:57:53.490 INFO:tasks.rados.rados.0.smithi033.stdout:2099: snap_remove snap 174 2024-09-21T09:57:54.150 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:53 smithi105 ceph-mon[86649]: pgmap v296: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 7.3 MiB/s wr, 35 op/s 2024-09-21T09:57:54.150 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:53 smithi105 ceph-mon[86649]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:57:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:53 smithi033 ceph-mon[98546]: pgmap v296: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 7.3 MiB/s wr, 35 op/s 2024-09-21T09:57:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:53 smithi033 ceph-mon[98546]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:57:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:53 smithi033 ceph-mon[96568]: pgmap v296: 121 pgs: 121 active+clean; 194 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 7.3 MiB/s wr, 35 op/s 2024-09-21T09:57:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:53 smithi033 ceph-mon[96568]: osdmap e446: 8 total, 8 up, 8 in 2024-09-21T09:57:54.443 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:54 smithi105 systemd[1]: Stopping Ceph osd.4 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:57:54.443 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:54.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:54.493 INFO:tasks.rados.rados.0.smithi033.stdout:2099: done (1 left) 2024-09-21T09:57:54.494 INFO:tasks.rados.rados.0.smithi033.stdout:2100: rollback oid 12 current snap is 188 2024-09-21T09:57:54.494 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 12 to 173 2024-09-21T09:57:54.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1346 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:57:54.494 INFO:tasks.rados.rados.0.smithi033.stdout:2098: done (1 left) 2024-09-21T09:57:54.494 INFO:tasks.rados.rados.0.smithi033.stdout:2101: read oid 11 snap -1 2024-09-21T09:57:54.494 INFO:tasks.rados.rados.0.smithi033.stdout:2101: expect (ObjNum 663 snap 184 seq_num 663) 2024-09-21T09:57:54.494 INFO:tasks.rados.rados.0.smithi033.stdout:2102: snap_create 2024-09-21T09:57:54.506 INFO:tasks.rados.rados.0.smithi033.stdout:2100: finishing rollback tid 0 to smithi03385152-12 2024-09-21T09:57:54.506 INFO:tasks.rados.rados.0.smithi033.stdout:2100: finishing rollback tid 1 to smithi03385152-12 2024-09-21T09:57:54.507 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1101 (ObjNum 129 snap 0 seq_num 91510720) dirty dne 2024-09-21T09:57:54.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[36605]: 2024-09-21T09:57:54.440+0000 7f21fdf58640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:57:54.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[36605]: 2024-09-21T09:57:54.440+0000 7f21fdf58640 -1 osd.4 446 *** Got signal Terminated *** 2024-09-21T09:57:54.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:54 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[36605]: 2024-09-21T09:57:54.440+0000 7f21fdf58640 -1 osd.4 446 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:57:54.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:54 smithi033 ceph-mon[98546]: osd.4 marked itself down and dead 2024-09-21T09:57:54.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:54 smithi033 ceph-mon[98546]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:57:54.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:54 smithi033 ceph-mon[98546]: osdmap e447: 8 total, 7 up, 8 in 2024-09-21T09:57:54.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:54 smithi033 ceph-mon[96568]: osd.4 marked itself down and dead 2024-09-21T09:57:54.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:54 smithi033 ceph-mon[96568]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:57:54.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:54 smithi033 ceph-mon[96568]: osdmap e447: 8 total, 7 up, 8 in 2024-09-21T09:57:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:54 smithi105 ceph-mon[86649]: osd.4 marked itself down and dead 2024-09-21T09:57:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:54 smithi105 ceph-mon[86649]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:57:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:54 smithi105 ceph-mon[86649]: osdmap e447: 8 total, 7 up, 8 in 2024-09-21T09:57:55.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:55 smithi105 podman[91721]: 2024-09-21 09:57:55.043811199 +0000 UTC m=+0.735871919 container died 02f11869fd8271c9bb28e4239e9bc5670d5b0141b88dd05864dfd66322b7ad5a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-09-21T09:57:55.490 INFO:tasks.rados.rados.0.smithi033.stdout:2100: done (2 left) 2024-09-21T09:57:55.491 INFO:tasks.rados.rados.0.smithi033.stdout:2102: done (1 left) 2024-09-21T09:57:55.491 INFO:tasks.rados.rados.0.smithi033.stdout:2103: copy_from oid 45 from oid 50 current snap is 189 2024-09-21T09:57:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:2104: read oid 35 snap -1 2024-09-21T09:57:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:2104: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:2105: read oid 32 snap -1 2024-09-21T09:57:55.493 INFO:tasks.rados.rados.0.smithi033.stdout:2105: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:55.494 INFO:tasks.rados.rados.0.smithi033.stdout:2106: read oid 8 snap -1 2024-09-21T09:57:55.494 INFO:tasks.rados.rados.0.smithi033.stdout:2106: expect deleted 2024-09-21T09:57:55.495 INFO:tasks.rados.rados.0.smithi033.stdout:2107: rollback oid 34 current snap is 189 2024-09-21T09:57:55.495 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 176 2024-09-21T09:57:55.495 INFO:tasks.rados.rados.0.smithi033.stdout:2108: snap_create 2024-09-21T09:57:55.500 INFO:tasks.rados.rados.0.smithi033.stdout:2103: finishing copy_from to smithi03385152-45 2024-09-21T09:57:55.501 INFO:tasks.rados.rados.0.smithi033.stdout:2103: got expected ENOENT (src dne) 2024-09-21T09:57:55.501 INFO:tasks.rados.rados.0.smithi033.stdout:2103: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:57:55.501 INFO:tasks.rados.rados.0.smithi033.stdout:2107: finishing rollback tid 1 to smithi03385152-34 2024-09-21T09:57:55.501 INFO:tasks.rados.rados.0.smithi033.stdout:2107: finishing rollback tid 2 to smithi03385152-34 2024-09-21T09:57:55.501 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1120 (ObjNum 635 snap 175 seq_num 635) dirty exists 2024-09-21T09:57:55.508 INFO:tasks.rados.rados.0.smithi033.stdout:2104: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:55.593 INFO:tasks.rados.rados.0.smithi033.stdout:2105: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:57:55.675 INFO:tasks.rados.rados.0.smithi033.stdout:2101: expect (ObjNum 663 snap 184 seq_num 663) 2024-09-21T09:57:55.742 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:55 smithi105 podman[91721]: 2024-09-21 09:57:55.514495948 +0000 UTC m=+1.206556658 container cleanup 02f11869fd8271c9bb28e4239e9bc5670d5b0141b88dd05864dfd66322b7ad5a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-21T09:57:55.742 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:55 smithi105 bash[91721]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4 2024-09-21T09:57:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:55 smithi105 ceph-mon[86649]: pgmap v299: 121 pgs: 121 active+clean; 196 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 7.3 MiB/s wr, 26 op/s 2024-09-21T09:57:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:55 smithi105 ceph-mon[86649]: osdmap e448: 8 total, 7 up, 8 in 2024-09-21T09:57:56.030 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:55 smithi105 podman[91741]: 2024-09-21 09:57:55.790855484 +0000 UTC m=+0.745700626 container remove 02f11869fd8271c9bb28e4239e9bc5670d5b0141b88dd05864dfd66322b7ad5a (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-21T09:57:56.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:55.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:56.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:55 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:55.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:55 smithi033 ceph-mon[98546]: pgmap v299: 121 pgs: 121 active+clean; 196 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 7.3 MiB/s wr, 26 op/s 2024-09-21T09:57:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:55 smithi033 ceph-mon[98546]: osdmap e448: 8 total, 7 up, 8 in 2024-09-21T09:57:56.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:55 smithi033 ceph-mon[96568]: pgmap v299: 121 pgs: 121 active+clean; 196 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 7.3 MiB/s wr, 26 op/s 2024-09-21T09:57:56.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:55 smithi033 ceph-mon[96568]: osdmap e448: 8 total, 7 up, 8 in 2024-09-21T09:57:56.412 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:56 smithi105 podman[91836]: 2024-09-21 09:57:56.116732176 +0000 UTC m=+0.099611316 container create 2a70363ee9ac488181129f80544c14a2147e437b62ae2af517e9834a25d32e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:57:56.412 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:56 smithi105 podman[91836]: 2024-09-21 09:57:56.032830227 +0000 UTC m=+0.015709259 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:57:56.412 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:56 smithi105 podman[91836]: 2024-09-21 09:57:56.318994035 +0000 UTC m=+0.301873068 container init 2a70363ee9ac488181129f80544c14a2147e437b62ae2af517e9834a25d32e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS) 2024-09-21T09:57:56.412 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:56 smithi105 podman[91836]: 2024-09-21 09:57:56.323630668 +0000 UTC m=+0.306509696 container start 2a70363ee9ac488181129f80544c14a2147e437b62ae2af517e9834a25d32e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T09:57:56.489 INFO:tasks.rados.rados.0.smithi033.stdout:2101: done (6 left) 2024-09-21T09:57:56.489 INFO:tasks.rados.rados.0.smithi033.stdout:2103: done (5 left) 2024-09-21T09:57:56.489 INFO:tasks.rados.rados.0.smithi033.stdout:2104: done (4 left) 2024-09-21T09:57:56.489 INFO:tasks.rados.rados.0.smithi033.stdout:2105: done (3 left) 2024-09-21T09:57:56.489 INFO:tasks.rados.rados.0.smithi033.stdout:2106: done (2 left) 2024-09-21T09:57:56.489 INFO:tasks.rados.rados.0.smithi033.stdout:2107: done (1 left) 2024-09-21T09:57:56.489 INFO:tasks.rados.rados.0.smithi033.stdout:2108: done (0 left) 2024-09-21T09:57:56.489 INFO:tasks.rados.rados.0.smithi033.stdout:2109: delete oid 22 current snap is 190 2024-09-21T09:57:56.493 INFO:tasks.rados.rados.0.smithi033.stdout:2109: done (0 left) 2024-09-21T09:57:56.493 INFO:tasks.rados.rados.0.smithi033.stdout:2110: copy_from oid 7 from oid 41 current snap is 190 2024-09-21T09:57:56.493 INFO:tasks.rados.rados.0.smithi033.stdout:2111: read oid 34 snap -1 2024-09-21T09:57:56.493 INFO:tasks.rados.rados.0.smithi033.stdout:2111: expect (ObjNum 635 snap 175 seq_num 635) 2024-09-21T09:57:56.493 INFO:tasks.rados.rados.0.smithi033.stdout:2112: delete oid 6 current snap is 190 2024-09-21T09:57:56.496 INFO:tasks.rados.rados.0.smithi033.stdout:2112: done (2 left) 2024-09-21T09:57:56.496 INFO:tasks.rados.rados.0.smithi033.stdout:2113: snap_create 2024-09-21T09:57:56.499 INFO:tasks.rados.rados.0.smithi033.stdout:2111: expect (ObjNum 635 snap 175 seq_num 635) 2024-09-21T09:57:56.528 INFO:tasks.rados.rados.0.smithi033.stdout:2110: finishing copy_from racing read to smithi03385152-7 2024-09-21T09:57:56.528 INFO:tasks.rados.rados.0.smithi033.stdout:2110: finishing copy_from to smithi03385152-7 2024-09-21T09:57:56.528 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 761 (ObjNum 651 snap 179 seq_num 651) dirty exists 2024-09-21T09:57:56.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:56 smithi105 podman[91836]: 2024-09-21 09:57:56.410592849 +0000 UTC m=+0.393471896 container attach 2a70363ee9ac488181129f80544c14a2147e437b62ae2af517e9834a25d32e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:57:56.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:56 smithi105 podman[91836]: 2024-09-21 09:57:56.519345447 +0000 UTC m=+0.502224473 container died 2a70363ee9ac488181129f80544c14a2147e437b62ae2af517e9834a25d32e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T09:57:57.199 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:57:56 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:57:56.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:57:57.488 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 podman[91878]: 2024-09-21 09:57:57.197984738 +0000 UTC m=+0.672853840 container remove 2a70363ee9ac488181129f80544c14a2147e437b62ae2af517e9834a25d32e17 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-21T09:57:57.488 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.4.service: Deactivated successfully. 2024-09-21T09:57:57.489 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 systemd[1]: Stopped Ceph osd.4 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:57:57.489 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.4.service: Consumed 13.337s CPU time. 2024-09-21T09:57:57.489 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 systemd[1]: Starting Ceph osd.4 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:57:57.502 INFO:tasks.rados.rados.0.smithi033.stdout:2110: done (2 left) 2024-09-21T09:57:57.502 INFO:tasks.rados.rados.0.smithi033.stdout:2111: done (1 left) 2024-09-21T09:57:57.502 INFO:tasks.rados.rados.0.smithi033.stdout:2113: done (0 left) 2024-09-21T09:57:57.502 INFO:tasks.rados.rados.0.smithi033.stdout:2114: copy_from oid 23 from oid 20 current snap is 191 2024-09-21T09:57:57.502 INFO:tasks.rados.rados.0.smithi033.stdout:2115: rmattr oid 4 current snap is 191 2024-09-21T09:57:57.502 INFO:tasks.rados.rados.0.smithi033.stdout:2115: done (1 left) 2024-09-21T09:57:57.502 INFO:tasks.rados.rados.0.smithi033.stdout:2116: snap_create 2024-09-21T09:57:57.505 INFO:tasks.rados.rados.0.smithi033.stdout:2114: finishing copy_from to smithi03385152-23 2024-09-21T09:57:57.505 INFO:tasks.rados.rados.0.smithi033.stdout:2114: got expected ENOENT (src dne) 2024-09-21T09:57:57.505 INFO:tasks.rados.rados.0.smithi033.stdout:2114: finishing copy_from racing read to smithi03385152-23 2024-09-21T09:57:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:57 smithi033 ceph-mon[98546]: pgmap v302: 121 pgs: 23 active+undersized, 7 peering, 2 stale+active+clean, 7 active+undersized+degraded, 82 active+clean; 195 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 452 KiB/s wr, 9 op/s; 32/408 objects degraded (7.843%) 2024-09-21T09:57:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:57 smithi033 ceph-mon[98546]: osdmap e449: 8 total, 7 up, 8 in 2024-09-21T09:57:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:57 smithi033 ceph-mon[98546]: Health check failed: Degraded data redundancy: 32/408 objects degraded (7.843%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:57.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:57 smithi033 ceph-mon[96568]: pgmap v302: 121 pgs: 23 active+undersized, 7 peering, 2 stale+active+clean, 7 active+undersized+degraded, 82 active+clean; 195 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 452 KiB/s wr, 9 op/s; 32/408 objects degraded (7.843%) 2024-09-21T09:57:57.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:57 smithi033 ceph-mon[96568]: osdmap e449: 8 total, 7 up, 8 in 2024-09-21T09:57:57.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:57 smithi033 ceph-mon[96568]: Health check failed: Degraded data redundancy: 32/408 objects degraded (7.843%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:57.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:57 smithi105 ceph-mon[86649]: pgmap v302: 121 pgs: 23 active+undersized, 7 peering, 2 stale+active+clean, 7 active+undersized+degraded, 82 active+clean; 195 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 452 KiB/s wr, 9 op/s; 32/408 objects degraded (7.843%) 2024-09-21T09:57:57.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:57 smithi105 ceph-mon[86649]: osdmap e449: 8 total, 7 up, 8 in 2024-09-21T09:57:57.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:57 smithi105 ceph-mon[86649]: Health check failed: Degraded data redundancy: 32/408 objects degraded (7.843%), 7 pgs degraded (PG_DEGRADED) 2024-09-21T09:57:57.777 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 podman[91981]: 2024-09-21 09:57:57.542834854 +0000 UTC m=+0.015553931 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:57:57.777 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 podman[91981]: 2024-09-21 09:57:57.649018378 +0000 UTC m=+0.121737490 container create ed06c498009abd5b675b025fc3bb71e9b4d1151ccbac292b788eaf89e46116a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-21T09:57:58.027 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 podman[91981]: 2024-09-21 09:57:57.926500281 +0000 UTC m=+0.399219356 container init ed06c498009abd5b675b025fc3bb71e9b4d1151ccbac292b788eaf89e46116a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T09:57:58.028 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:57 smithi105 podman[91981]: 2024-09-21 09:57:57.931523438 +0000 UTC m=+0.404242509 container start ed06c498009abd5b675b025fc3bb71e9b4d1151ccbac292b788eaf89e46116a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-62da54b) 2024-09-21T09:57:58.028 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 podman[91981]: 2024-09-21 09:57:58.026437109 +0000 UTC m=+0.499156199 container attach ed06c498009abd5b675b025fc3bb71e9b4d1151ccbac292b788eaf89e46116a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True) 2024-09-21T09:57:58.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:57:58.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:57:58.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:57:58.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:57:58.481 INFO:tasks.rados.rados.0.smithi033.stdout:2114: done (1 left) 2024-09-21T09:57:58.482 INFO:tasks.rados.rados.0.smithi033.stdout:2116: done (0 left) 2024-09-21T09:57:58.482 INFO:tasks.rados.rados.0.smithi033.stdout:2117: rollback oid 37 current snap is 192 2024-09-21T09:57:58.482 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 190 2024-09-21T09:57:58.482 INFO:tasks.rados.rados.0.smithi033.stdout:2118: read oid 25 snap -1 2024-09-21T09:57:58.482 INFO:tasks.rados.rados.0.smithi033.stdout:2118: expect (ObjNum 535 snap 143 seq_num 535) 2024-09-21T09:57:58.482 INFO:tasks.rados.rados.0.smithi033.stdout:2119: setattr oid 23 current snap is 192 2024-09-21T09:57:58.483 INFO:tasks.rados.rados.0.smithi033.stdout:2120: delete oid 38 current snap is 192 2024-09-21T09:57:58.491 INFO:tasks.rados.rados.0.smithi033.stdout:2117: finishing rollback tid 0 to smithi03385152-37 2024-09-21T09:57:58.491 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1017 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:57:58.492 INFO:tasks.rados.rados.0.smithi033.stdout:2117: finishing rollback tid 1 to smithi03385152-37 2024-09-21T09:57:58.492 INFO:tasks.rados.rados.0.smithi033.stdout:2117: finishing rollback tid 2 to smithi03385152-37 2024-09-21T09:57:58.492 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1021 (ObjNum 503 snap 132 seq_num 503) dirty exists 2024-09-21T09:57:58.493 INFO:tasks.rados.rados.0.smithi033.stdout:2117: done (3 left) 2024-09-21T09:57:58.493 INFO:tasks.rados.rados.0.smithi033.stdout:2119: done (2 left) 2024-09-21T09:57:58.493 INFO:tasks.rados.rados.0.smithi033.stdout:2120: done (1 left) 2024-09-21T09:57:58.494 INFO:tasks.rados.rados.0.smithi033.stdout:2121: read oid 40 snap -1 2024-09-21T09:57:58.494 INFO:tasks.rados.rados.0.smithi033.stdout:2121: expect deleted 2024-09-21T09:57:58.494 INFO:tasks.rados.rados.0.smithi033.stdout:2122: read oid 2 snap 178 2024-09-21T09:57:58.494 INFO:tasks.rados.rados.0.smithi033.stdout:2122: expect (ObjNum 90773056 snap 21955 seq_num 91008496) 2024-09-21T09:57:58.494 INFO:tasks.rados.rados.0.smithi033.stdout:2123: delete oid 47 current snap is 192 2024-09-21T09:57:58.501 INFO:tasks.rados.rados.0.smithi033.stdout:2118: expect (ObjNum 535 snap 143 seq_num 535) 2024-09-21T09:57:58.587 INFO:tasks.rados.rados.0.smithi033.stdout:2122: expect (ObjNum 90773056 snap 21955 seq_num 91008496) 2024-09-21T09:57:58.589 INFO:tasks.rados.rados.0.smithi033.stdout:2118: done (3 left) 2024-09-21T09:57:58.589 INFO:tasks.rados.rados.0.smithi033.stdout:2121: done (2 left) 2024-09-21T09:57:58.589 INFO:tasks.rados.rados.0.smithi033.stdout:2122: done (1 left) 2024-09-21T09:57:58.589 INFO:tasks.rados.rados.0.smithi033.stdout:2123: done (0 left) 2024-09-21T09:57:58.590 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 192 2024-09-21T09:57:58.590 INFO:tasks.rados.rados.0.smithi033.stdout:2124: seq_num 689 ranges {6733824=745472,7479296=786432,8265728=417792} 2024-09-21T09:57:58.630 INFO:tasks.rados.rados.0.smithi033.stdout:2124: writing smithi03385152-23 from 6733824 to 7479296 tid 1 2024-09-21T09:57:58.634 INFO:tasks.rados.rados.0.smithi033.stdout:2124: writing smithi03385152-23 from 7479296 to 8265728 tid 2 2024-09-21T09:57:58.637 INFO:tasks.rados.rados.0.smithi033.stdout:2124: writing smithi03385152-23 from 8265728 to 8683520 tid 3 2024-09-21T09:57:58.637 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 192 2024-09-21T09:57:58.637 INFO:tasks.rados.rados.0.smithi033.stdout:2124: finishing write tid 1 to smithi03385152-23 2024-09-21T09:57:58.637 INFO:tasks.rados.rados.0.smithi033.stdout:2124: finishing write tid 2 to smithi03385152-23 2024-09-21T09:57:58.638 INFO:tasks.rados.rados.0.smithi033.stdout:2125: seq_num 690 ranges {1761280=761856,2523136=663552,3186688=409600,3596288=131072} 2024-09-21T09:57:58.651 INFO:tasks.rados.rados.0.smithi033.stdout:2125: writing smithi03385152-43 from 1761280 to 2523136 tid 1 2024-09-21T09:57:58.654 INFO:tasks.rados.rados.0.smithi033.stdout:2125: writing smithi03385152-43 from 2523136 to 3186688 tid 2 2024-09-21T09:57:58.657 INFO:tasks.rados.rados.0.smithi033.stdout:2125: writing smithi03385152-43 from 3186688 to 3596288 tid 3 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:2125: writing smithi03385152-43 from 3596288 to 3727360 tid 4 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 192 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:2124: finishing write tid 3 to smithi03385152-23 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:2124: finishing write tid 4 to smithi03385152-23 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:2124: finishing write tid 5 to smithi03385152-23 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1021 (ObjNum 689 snap 192 seq_num 689) dirty exists 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:2124: left oid 23 (ObjNum 689 snap 192 seq_num 689) 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:2125: finishing write tid 1 to smithi03385152-43 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:2125: finishing write tid 2 to smithi03385152-43 2024-09-21T09:57:58.659 INFO:tasks.rados.rados.0.smithi033.stdout:2126: seq_num 691 ranges {0=778240,778240=548864,1327104=614400,1941504=40960} 2024-09-21T09:57:58.663 INFO:tasks.rados.rados.0.smithi033.stdout:2126: writing smithi03385152-48 from 0 to 778240 tid 1 2024-09-21T09:57:58.666 INFO:tasks.rados.rados.0.smithi033.stdout:2126: writing smithi03385152-48 from 778240 to 1327104 tid 2 2024-09-21T09:57:58.669 INFO:tasks.rados.rados.0.smithi033.stdout:2126: writing smithi03385152-48 from 1327104 to 1941504 tid 3 2024-09-21T09:57:58.670 INFO:tasks.rados.rados.0.smithi033.stdout:2126: writing smithi03385152-48 from 1941504 to 1982464 tid 4 2024-09-21T09:57:58.670 INFO:tasks.rados.rados.0.smithi033.stdout:2124: done (2 left) 2024-09-21T09:57:58.670 INFO:tasks.rados.rados.0.smithi033.stdout:2127: read oid 11 snap -1 2024-09-21T09:57:58.670 INFO:tasks.rados.rados.0.smithi033.stdout:2127: expect (ObjNum 663 snap 184 seq_num 663) 2024-09-21T09:57:58.670 INFO:tasks.rados.rados.0.smithi033.stdout:2128: snap_remove snap 177 2024-09-21T09:57:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-mon[86649]: osdmap e450: 8 total, 7 up, 8 in 2024-09-21T09:57:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-mon[86649]: osdmap e451: 8 total, 7 up, 8 in 2024-09-21T09:57:58.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T09:57:58.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T09:57:58.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:57:58.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-21T09:57:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:58 smithi033 ceph-mon[98546]: osdmap e450: 8 total, 7 up, 8 in 2024-09-21T09:57:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:58 smithi033 ceph-mon[98546]: osdmap e451: 8 total, 7 up, 8 in 2024-09-21T09:57:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:58 smithi033 ceph-mon[96568]: osdmap e450: 8 total, 7 up, 8 in 2024-09-21T09:57:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:58 smithi033 ceph-mon[96568]: osdmap e451: 8 total, 7 up, 8 in 2024-09-21T09:57:59.139 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:57:59.139 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-21T09:57:59.139 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:57:59.139 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-21T09:57:59.139 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T09:57:59.140 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-21T09:57:59.140 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T09:57:59.140 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-21T09:57:59.140 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate[91996]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T09:57:59.140 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 bash[91981]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-21T09:57:59.140 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:58 smithi105 podman[91981]: 2024-09-21 09:57:58.844351651 +0000 UTC m=+1.317070721 container died ed06c498009abd5b675b025fc3bb71e9b4d1151ccbac292b788eaf89e46116a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T09:57:59.511 INFO:tasks.rados.rados.0.smithi033.stdout:2128: done (3 left) 2024-09-21T09:57:59.511 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 192 2024-09-21T09:57:59.511 INFO:tasks.rados.rados.0.smithi033.stdout:2129: seq_num 692 ranges {1310720=794624,2105344=483328,2588672=720896} 2024-09-21T09:57:59.523 INFO:tasks.rados.rados.0.smithi033.stdout:2129: writing smithi03385152-41 from 1310720 to 2105344 tid 1 2024-09-21T09:57:59.526 INFO:tasks.rados.rados.0.smithi033.stdout:2129: writing smithi03385152-41 from 2105344 to 2588672 tid 2 2024-09-21T09:57:59.533 INFO:tasks.rados.rados.0.smithi033.stdout:2129: writing smithi03385152-41 from 2588672 to 3309568 tid 3 2024-09-21T09:57:59.533 INFO:tasks.rados.rados.0.smithi033.stdout:2130: read oid 8 snap 189 2024-09-21T09:57:59.533 INFO:tasks.rados.rados.0.smithi033.stdout:2130: expect deleted 2024-09-21T09:57:59.533 INFO:tasks.rados.rados.0.smithi033.stdout:2131: read oid 50 snap -1 2024-09-21T09:57:59.533 INFO:tasks.rados.rados.0.smithi033.stdout:2131: expect deleted 2024-09-21T09:57:59.533 INFO:tasks.rados.rados.0.smithi033.stdout:2132: rollback oid 25 current snap is 192 2024-09-21T09:57:59.534 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 25 to 186 2024-09-21T09:57:59.534 INFO:tasks.rados.rados.0.smithi033.stdout:2125: finishing write tid 3 to smithi03385152-43 2024-09-21T09:57:59.534 INFO:tasks.rados.rados.0.smithi033.stdout:2125: finishing write tid 4 to smithi03385152-43 2024-09-21T09:57:59.534 INFO:tasks.rados.rados.0.smithi033.stdout:2133: read oid 34 snap -1 2024-09-21T09:57:59.534 INFO:tasks.rados.rados.0.smithi033.stdout:2133: expect (ObjNum 635 snap 175 seq_num 635) 2024-09-21T09:57:59.534 INFO:tasks.rados.rados.0.smithi033.stdout:2134: rollback oid 10 current snap is 192 2024-09-21T09:57:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 188 2024-09-21T09:57:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:2125: finishing write tid 5 to smithi03385152-43 2024-09-21T09:57:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:2125: finishing write tid 6 to smithi03385152-43 2024-09-21T09:57:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1026 (ObjNum 690 snap 192 seq_num 690) dirty exists 2024-09-21T09:57:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:2125: left oid 43 (ObjNum 690 snap 192 seq_num 690) 2024-09-21T09:57:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:2125: done (8 left) 2024-09-21T09:57:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:2135: delete oid 12 current snap is 192 2024-09-21T09:57:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:2126: finishing write tid 1 to smithi03385152-48 2024-09-21T09:57:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2126: finishing write tid 2 to smithi03385152-48 2024-09-21T09:57:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2126: finishing write tid 3 to smithi03385152-48 2024-09-21T09:57:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2126: finishing write tid 4 to smithi03385152-48 2024-09-21T09:57:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2126: finishing write tid 5 to smithi03385152-48 2024-09-21T09:57:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2126: finishing write tid 6 to smithi03385152-48 2024-09-21T09:57:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 1128 (ObjNum 691 snap 192 seq_num 691) dirty exists 2024-09-21T09:57:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2126: left oid 48 (ObjNum 691 snap 192 seq_num 691) 2024-09-21T09:57:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2127: expect (ObjNum 663 snap 184 seq_num 663) 2024-09-21T09:57:59.775 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:57:59 smithi105 ceph-mon[86649]: pgmap v305: 121 pgs: 27 active+undersized, 7 peering, 1 stale+active+clean, 9 active+undersized+degraded, 77 active+clean; 197 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 456 KiB/s wr, 8 op/s; 40/411 objects degraded (9.732%) 2024-09-21T09:57:59.775 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 podman[92156]: 2024-09-21 09:57:59.438623737 +0000 UTC m=+0.588536118 container remove ed06c498009abd5b675b025fc3bb71e9b4d1151ccbac292b788eaf89e46116a1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-activate, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T09:57:59.775 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 podman[92220]: 2024-09-21 09:57:59.550402217 +0000 UTC m=+0.019093196 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:57:59.775 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 podman[92220]: 2024-09-21 09:57:59.656042172 +0000 UTC m=+0.124733149 container create 01f4f66dbe7202ac8254b91223c9bbf298c172f91d3166bdeab7eff23ecbbea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, io.buildah.version=1.37.2) 2024-09-21T09:57:59.917 INFO:tasks.rados.rados.0.smithi033.stdout:2129: finishing write tid 1 to smithi03385152-41 2024-09-21T09:57:59.917 INFO:tasks.rados.rados.0.smithi033.stdout:2129: finishing write tid 2 to smithi03385152-41 2024-09-21T09:57:59.917 INFO:tasks.rados.rados.0.smithi033.stdout:2126: done (8 left) 2024-09-21T09:57:59.917 INFO:tasks.rados.rados.0.smithi033.stdout:2127: done (7 left) 2024-09-21T09:57:59.917 INFO:tasks.rados.rados.0.smithi033.stdout:2130: done (6 left) 2024-09-21T09:57:59.917 INFO:tasks.rados.rados.0.smithi033.stdout:2131: done (5 left) 2024-09-21T09:57:59.918 INFO:tasks.rados.rados.0.smithi033.stdout:2135: done (4 left) 2024-09-21T09:57:59.918 INFO:tasks.rados.rados.0.smithi033.stdout:2136: read oid 20 snap -1 2024-09-21T09:57:59.918 INFO:tasks.rados.rados.0.smithi033.stdout:2136: expect deleted 2024-09-21T09:57:59.918 INFO:tasks.rados.rados.0.smithi033.stdout:2137: copy_from oid 29 from oid 21 current snap is 192 2024-09-21T09:57:59.918 INFO:tasks.rados.rados.0.smithi033.stdout:2133: expect (ObjNum 635 snap 175 seq_num 635) 2024-09-21T09:57:59.944 INFO:tasks.rados.rados.0.smithi033.stdout:2134: finishing rollback tid 0 to smithi03385152-10 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2134: finishing rollback tid 1 to smithi03385152-10 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2134: finishing rollback tid 2 to smithi03385152-10 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1017 (ObjNum 508 snap 135 seq_num 508) dirty exists 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2129: finishing write tid 3 to smithi03385152-41 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2129: finishing write tid 4 to smithi03385152-41 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2132: finishing rollback tid 0 to smithi03385152-25 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2132: finishing rollback tid 1 to smithi03385152-25 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2132: finishing rollback tid 2 to smithi03385152-25 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 915 (ObjNum 535 snap 143 seq_num 535) dirty exists 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2132: done (5 left) 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2133: done (4 left) 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2134: done (3 left) 2024-09-21T09:57:59.945 INFO:tasks.rados.rados.0.smithi033.stdout:2138: snap_create 2024-09-21T09:57:59.946 INFO:tasks.rados.rados.0.smithi033.stdout:2129: finishing write tid 5 to smithi03385152-41 2024-09-21T09:57:59.946 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 770 (ObjNum 692 snap 192 seq_num 692) dirty exists 2024-09-21T09:57:59.946 INFO:tasks.rados.rados.0.smithi033.stdout:2129: left oid 41 (ObjNum 692 snap 192 seq_num 692) 2024-09-21T09:57:59.971 INFO:tasks.rados.rados.0.smithi033.stdout:2137: finishing copy_from racing read to smithi03385152-29 2024-09-21T09:57:59.976 INFO:tasks.rados.rados.0.smithi033.stdout:2137: finishing copy_from to smithi03385152-29 2024-09-21T09:57:59.976 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1016 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:57:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:57:59 smithi033 ceph-mon[96568]: pgmap v305: 121 pgs: 27 active+undersized, 7 peering, 1 stale+active+clean, 9 active+undersized+degraded, 77 active+clean; 197 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 456 KiB/s wr, 8 op/s; 40/411 objects degraded (9.732%) 2024-09-21T09:57:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:57:59 smithi033 ceph-mon[98546]: pgmap v305: 121 pgs: 27 active+undersized, 7 peering, 1 stale+active+clean, 9 active+undersized+degraded, 77 active+clean; 197 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 456 KiB/s wr, 8 op/s; 40/411 objects degraded (9.732%) 2024-09-21T09:58:00.215 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 podman[92220]: 2024-09-21 09:57:59.85816785 +0000 UTC m=+0.326858816 container init 01f4f66dbe7202ac8254b91223c9bbf298c172f91d3166bdeab7eff23ecbbea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-62da54b) 2024-09-21T09:58:00.215 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 podman[92220]: 2024-09-21 09:57:59.863069644 +0000 UTC m=+0.331760610 container start 01f4f66dbe7202ac8254b91223c9bbf298c172f91d3166bdeab7eff23ecbbea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-21T09:58:00.215 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 ceph-osd[92258]: -- 172.21.15.105:0/2449893360 <== mon.1 v2:172.21.15.105:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x564ad47481a0 con 0x564ad3915c00 2024-09-21T09:58:00.215 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 ceph-osd[92258]: -- 172.21.15.105:0/2449893360 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564ad3937a40 con 0x564ad3915c00 2024-09-21T09:58:00.215 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 bash[92220]: 01f4f66dbe7202ac8254b91223c9bbf298c172f91d3166bdeab7eff23ecbbea1 2024-09-21T09:58:00.216 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:57:59 smithi105 systemd[1]: Started Ceph osd.4 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:58:00.507 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:00.518 INFO:tasks.rados.rados.0.smithi033.stdout:2129: done (3 left) 2024-09-21T09:58:00.518 INFO:tasks.rados.rados.0.smithi033.stdout:2136: done (2 left) 2024-09-21T09:58:00.518 INFO:tasks.rados.rados.0.smithi033.stdout:2137: done (1 left) 2024-09-21T09:58:00.518 INFO:tasks.rados.rados.0.smithi033.stdout:2138: done (0 left) 2024-09-21T09:58:00.518 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 193 2024-09-21T09:58:00.518 INFO:tasks.rados.rados.0.smithi033.stdout:2139: seq_num 694 ranges {2277376=425984,2703360=655360,3358720=786432,4145152=163840} 2024-09-21T09:58:00.532 INFO:tasks.rados.rados.0.smithi033.stdout:2139: writing smithi03385152-36 from 2277376 to 2703360 tid 1 2024-09-21T09:58:00.536 INFO:tasks.rados.rados.0.smithi033.stdout:2139: writing smithi03385152-36 from 2703360 to 3358720 tid 2 2024-09-21T09:58:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:2139: writing smithi03385152-36 from 3358720 to 4145152 tid 3 2024-09-21T09:58:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:2139: writing smithi03385152-36 from 4145152 to 4308992 tid 4 2024-09-21T09:58:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 193 2024-09-21T09:58:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:2139: finishing write tid 1 to smithi03385152-36 2024-09-21T09:58:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2140: seq_num 695 ranges {2310144=442368,2752512=540672,3293184=598016,3891200=466944} 2024-09-21T09:58:00.557 INFO:tasks.rados.rados.0.smithi033.stdout:2140: writing smithi03385152-31 from 2310144 to 2752512 tid 1 2024-09-21T09:58:00.560 INFO:tasks.rados.rados.0.smithi033.stdout:2140: writing smithi03385152-31 from 2752512 to 3293184 tid 2 2024-09-21T09:58:00.564 INFO:tasks.rados.rados.0.smithi033.stdout:2140: writing smithi03385152-31 from 3293184 to 3891200 tid 3 2024-09-21T09:58:00.566 INFO:tasks.rados.rados.0.smithi033.stdout:2140: writing smithi03385152-31 from 3891200 to 4358144 tid 4 2024-09-21T09:58:00.567 INFO:tasks.rados.rados.0.smithi033.stdout:2141: rmattr oid 39 current snap is 193 2024-09-21T09:58:00.567 INFO:tasks.rados.rados.0.smithi033.stdout:2141: done (2 left) 2024-09-21T09:58:00.567 INFO:tasks.rados.rados.0.smithi033.stdout:2142: snap_remove snap 191 2024-09-21T09:58:00.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:00 smithi105 ceph-mon[86649]: osdmap e452: 8 total, 7 up, 8 in 2024-09-21T09:58:00.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:00.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:00.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:00.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:00 smithi033 ceph-mon[96568]: osdmap e452: 8 total, 7 up, 8 in 2024-09-21T09:58:00.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:00.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:00.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:00.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:00 smithi033 ceph-mon[98546]: osdmap e452: 8 total, 7 up, 8 in 2024-09-21T09:58:00.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:00.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:00.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:01.526 INFO:tasks.rados.rados.0.smithi033.stdout:2142: done (2 left) 2024-09-21T09:58:01.526 INFO:tasks.rados.rados.0.smithi033.stdout:2143: delete oid 18 current snap is 193 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2139: finishing write tid 2 to smithi03385152-36 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2139: finishing write tid 3 to smithi03385152-36 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2139: finishing write tid 4 to smithi03385152-36 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2139: finishing write tid 5 to smithi03385152-36 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2139: finishing write tid 6 to smithi03385152-36 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1385 (ObjNum 694 snap 193 seq_num 694) dirty exists 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2139: left oid 36 (ObjNum 694 snap 193 seq_num 694) 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2140: finishing write tid 1 to smithi03385152-31 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2140: finishing write tid 2 to smithi03385152-31 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2140: finishing write tid 3 to smithi03385152-31 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2140: finishing write tid 4 to smithi03385152-31 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2140: finishing write tid 5 to smithi03385152-31 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:2140: finishing write tid 6 to smithi03385152-31 2024-09-21T09:58:01.527 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1391 (ObjNum 695 snap 193 seq_num 695) dirty exists 2024-09-21T09:58:01.528 INFO:tasks.rados.rados.0.smithi033.stdout:2140: left oid 31 (ObjNum 695 snap 193 seq_num 695) 2024-09-21T09:58:01.528 INFO:tasks.rados.rados.0.smithi033.stdout:2139: done (2 left) 2024-09-21T09:58:01.528 INFO:tasks.rados.rados.0.smithi033.stdout:2140: done (1 left) 2024-09-21T09:58:01.528 INFO:tasks.rados.rados.0.smithi033.stdout:2143: done (0 left) 2024-09-21T09:58:01.528 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 193 2024-09-21T09:58:01.528 INFO:tasks.rados.rados.0.smithi033.stdout:2144: seq_num 697 ranges {6733824=475136,7208960=704512,7913472=557056,8470528=344064} 2024-09-21T09:58:01.567 INFO:tasks.rados.rados.0.smithi033.stdout:2144: writing smithi03385152-27 from 6733824 to 7208960 tid 1 2024-09-21T09:58:01.571 INFO:tasks.rados.rados.0.smithi033.stdout:2144: writing smithi03385152-27 from 7208960 to 7913472 tid 2 2024-09-21T09:58:01.574 INFO:tasks.rados.rados.0.smithi033.stdout:2144: writing smithi03385152-27 from 7913472 to 8470528 tid 3 2024-09-21T09:58:01.576 INFO:tasks.rados.rados.0.smithi033.stdout:2144: writing smithi03385152-27 from 8470528 to 8814592 tid 4 2024-09-21T09:58:01.576 INFO:tasks.rados.rados.0.smithi033.stdout:2145: delete oid 7 current snap is 193 2024-09-21T09:58:01.576 INFO:tasks.rados.rados.0.smithi033.stdout:2144: finishing write tid 1 to smithi03385152-27 2024-09-21T09:58:01.576 INFO:tasks.rados.rados.0.smithi033.stdout:2144: finishing write tid 2 to smithi03385152-27 2024-09-21T09:58:01.578 INFO:tasks.rados.rados.0.smithi033.stdout:2144: finishing write tid 3 to smithi03385152-27 2024-09-21T09:58:01.578 INFO:tasks.rados.rados.0.smithi033.stdout:2144: finishing write tid 4 to smithi03385152-27 2024-09-21T09:58:01.579 INFO:tasks.rados.rados.0.smithi033.stdout:2145: done (1 left) 2024-09-21T09:58:01.579 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 193 2024-09-21T09:58:01.579 INFO:tasks.rados.rados.0.smithi033.stdout:2146: seq_num 699 ranges {0=516096,516096=475136,991232=573440,1564672=557056} 2024-09-21T09:58:01.581 INFO:tasks.rados.rados.0.smithi033.stdout:2146: writing smithi03385152-1 from 0 to 516096 tid 1 2024-09-21T09:58:01.584 INFO:tasks.rados.rados.0.smithi033.stdout:2146: writing smithi03385152-1 from 516096 to 991232 tid 2 2024-09-21T09:58:01.587 INFO:tasks.rados.rados.0.smithi033.stdout:2146: writing smithi03385152-1 from 991232 to 1564672 tid 3 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:2146: writing smithi03385152-1 from 1564672 to 2121728 tid 4 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:2147: snap_create 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:2144: finishing write tid 5 to smithi03385152-27 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:2144: finishing write tid 6 to smithi03385152-27 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1055 (ObjNum 697 snap 193 seq_num 697) dirty exists 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:2144: left oid 27 (ObjNum 697 snap 193 seq_num 697) 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:2146: finishing write tid 1 to smithi03385152-1 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:2146: finishing write tid 2 to smithi03385152-1 2024-09-21T09:58:01.591 INFO:tasks.rados.rados.0.smithi033.stdout:2146: finishing write tid 3 to smithi03385152-1 2024-09-21T09:58:01.592 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:01 smithi105 ceph-mon[86649]: pgmap v308: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 199 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 936 KiB/s wr, 7 op/s; 62/423 objects degraded (14.657%) 2024-09-21T09:58:01.592 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:01 smithi105 ceph-mon[86649]: osdmap e453: 8 total, 7 up, 8 in 2024-09-21T09:58:01.592 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:01.592 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:01.593 INFO:tasks.rados.rados.0.smithi033.stdout:2146: finishing write tid 4 to smithi03385152-1 2024-09-21T09:58:01.593 INFO:tasks.rados.rados.0.smithi033.stdout:2146: finishing write tid 5 to smithi03385152-1 2024-09-21T09:58:01.597 INFO:tasks.rados.rados.0.smithi033.stdout:2146: finishing write tid 6 to smithi03385152-1 2024-09-21T09:58:01.597 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 911 (ObjNum 699 snap 193 seq_num 699) dirty exists 2024-09-21T09:58:01.597 INFO:tasks.rados.rados.0.smithi033.stdout:2146: left oid 1 (ObjNum 699 snap 193 seq_num 699) 2024-09-21T09:58:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:58:01] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.51.0" 2024-09-21T09:58:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-mon[98546]: pgmap v308: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 199 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 936 KiB/s wr, 7 op/s; 62/423 objects degraded (14.657%) 2024-09-21T09:58:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-mon[98546]: osdmap e453: 8 total, 7 up, 8 in 2024-09-21T09:58:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-mon[96568]: pgmap v308: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 199 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.9 MiB/s rd, 936 KiB/s wr, 7 op/s; 62/423 objects degraded (14.657%) 2024-09-21T09:58:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-mon[96568]: osdmap e453: 8 total, 7 up, 8 in 2024-09-21T09:58:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:02.030 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:58:01 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T09:58:01.764+0000 7ff784a39640 -1 Falling back to public interface 2024-09-21T09:58:02.537 INFO:tasks.rados.rados.0.smithi033.stdout:2144: done (2 left) 2024-09-21T09:58:02.537 INFO:tasks.rados.rados.0.smithi033.stdout:2146: done (1 left) 2024-09-21T09:58:02.537 INFO:tasks.rados.rados.0.smithi033.stdout:2147: done (0 left) 2024-09-21T09:58:02.537 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 194 2024-09-21T09:58:02.537 INFO:tasks.rados.rados.0.smithi033.stdout:2148: seq_num 700 ranges {4358144=532480,4890624=409600,5300224=737280,6037504=458752} 2024-09-21T09:58:02.563 INFO:tasks.rados.rados.0.smithi033.stdout:2148: writing smithi03385152-31 from 4358144 to 4890624 tid 1 2024-09-21T09:58:02.566 INFO:tasks.rados.rados.0.smithi033.stdout:2148: writing smithi03385152-31 from 4890624 to 5300224 tid 2 2024-09-21T09:58:02.571 INFO:tasks.rados.rados.0.smithi033.stdout:2148: writing smithi03385152-31 from 5300224 to 6037504 tid 3 2024-09-21T09:58:02.574 INFO:tasks.rados.rados.0.smithi033.stdout:2148: writing smithi03385152-31 from 6037504 to 6496256 tid 4 2024-09-21T09:58:02.574 INFO:tasks.rados.rados.0.smithi033.stdout:2149: read oid 19 snap -1 2024-09-21T09:58:02.574 INFO:tasks.rados.rados.0.smithi033.stdout:2149: expect deleted 2024-09-21T09:58:02.574 INFO:tasks.rados.rados.0.smithi033.stdout:2150: snap_remove snap 176 2024-09-21T09:58:02.687 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:02 smithi105 ceph-mon[86649]: osdmap e454: 8 total, 7 up, 8 in 2024-09-21T09:58:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:02 smithi033 ceph-mon[98546]: osdmap e454: 8 total, 7 up, 8 in 2024-09-21T09:58:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:02 smithi033 ceph-mon[96568]: osdmap e454: 8 total, 7 up, 8 in 2024-09-21T09:58:03.482 INFO:tasks.rados.rados.0.smithi033.stdout:2148: finishing write tid 1 to smithi03385152-31 2024-09-21T09:58:03.482 INFO:tasks.rados.rados.0.smithi033.stdout:2148: finishing write tid 2 to smithi03385152-31 2024-09-21T09:58:03.482 INFO:tasks.rados.rados.0.smithi033.stdout:2148: finishing write tid 3 to smithi03385152-31 2024-09-21T09:58:03.482 INFO:tasks.rados.rados.0.smithi033.stdout:2148: finishing write tid 4 to smithi03385152-31 2024-09-21T09:58:03.482 INFO:tasks.rados.rados.0.smithi033.stdout:2148: finishing write tid 5 to smithi03385152-31 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:2148: finishing write tid 6 to smithi03385152-31 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1405 (ObjNum 700 snap 194 seq_num 700) dirty exists 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:2148: left oid 31 (ObjNum 700 snap 194 seq_num 700) 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:2148: done (2 left) 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:2149: done (1 left) 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:2150: done (0 left) 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:2151: rollback oid 2 current snap is 194 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 193 2024-09-21T09:58:03.483 INFO:tasks.rados.rados.0.smithi033.stdout:2152: setattr oid 36 current snap is 194 2024-09-21T09:58:03.484 INFO:tasks.rados.rados.0.smithi033.stdout:2153: snap_create 2024-09-21T09:58:03.492 INFO:tasks.rados.rados.0.smithi033.stdout:2151: finishing rollback tid 0 to smithi03385152-2 2024-09-21T09:58:03.492 INFO:tasks.rados.rados.0.smithi033.stdout:2151: finishing rollback tid 1 to smithi03385152-2 2024-09-21T09:58:03.493 INFO:tasks.rados.rados.0.smithi033.stdout:2151: finishing rollback tid 2 to smithi03385152-2 2024-09-21T09:58:03.493 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 918 (ObjNum 33 snap 0 seq_num 91322928) dirty exists 2024-09-21T09:58:03.494 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1407 (ObjNum 694 snap 193 seq_num 694) dirty exists 2024-09-21T09:58:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:03 smithi105 ceph-mon[86649]: pgmap v311: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 221 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.9 MiB/s wr, 22 op/s; 65/444 objects degraded (14.640%) 2024-09-21T09:58:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:03 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 65/444 objects degraded (14.640%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:03.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:03 smithi105 ceph-mon[86649]: osdmap e455: 8 total, 7 up, 8 in 2024-09-21T09:58:03.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:03 smithi105 ceph-mon[86649]: osdmap e456: 8 total, 7 up, 8 in 2024-09-21T09:58:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:03 smithi033 ceph-mon[96568]: pgmap v311: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 221 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.9 MiB/s wr, 22 op/s; 65/444 objects degraded (14.640%) 2024-09-21T09:58:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:03 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 65/444 objects degraded (14.640%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:03 smithi033 ceph-mon[96568]: osdmap e455: 8 total, 7 up, 8 in 2024-09-21T09:58:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:03 smithi033 ceph-mon[96568]: osdmap e456: 8 total, 7 up, 8 in 2024-09-21T09:58:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:03 smithi033 ceph-mon[98546]: pgmap v311: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 221 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.9 MiB/s wr, 22 op/s; 65/444 objects degraded (14.640%) 2024-09-21T09:58:03.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:03 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 65/444 objects degraded (14.640%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:03.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:03 smithi033 ceph-mon[98546]: osdmap e455: 8 total, 7 up, 8 in 2024-09-21T09:58:03.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:03 smithi033 ceph-mon[98546]: osdmap e456: 8 total, 7 up, 8 in 2024-09-21T09:58:04.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:04.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:58:04.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:04.147Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:04.482 INFO:tasks.rados.rados.0.smithi033.stdout:2151: done (2 left) 2024-09-21T09:58:04.482 INFO:tasks.rados.rados.0.smithi033.stdout:2152: done (1 left) 2024-09-21T09:58:04.482 INFO:tasks.rados.rados.0.smithi033.stdout:2153: done (0 left) 2024-09-21T09:58:04.482 INFO:tasks.rados.rados.0.smithi033.stdout:2154: delete oid 49 current snap is 195 2024-09-21T09:58:04.487 INFO:tasks.rados.rados.0.smithi033.stdout:2154: done (0 left) 2024-09-21T09:58:04.487 INFO:tasks.rados.rados.0.smithi033.stdout:2155: read oid 3 snap 189 2024-09-21T09:58:04.487 INFO:tasks.rados.rados.0.smithi033.stdout:2155: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:58:04.487 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 195 2024-09-21T09:58:04.487 INFO:tasks.rados.rados.0.smithi033.stdout:2156: seq_num 702 ranges {1318912=565248,1884160=581632,2465792=745472,3211264=278528} 2024-09-21T09:58:04.497 INFO:tasks.rados.rados.0.smithi033.stdout:2156: writing smithi03385152-10 from 1318912 to 1884160 tid 1 2024-09-21T09:58:04.500 INFO:tasks.rados.rados.0.smithi033.stdout:2156: writing smithi03385152-10 from 1884160 to 2465792 tid 2 2024-09-21T09:58:04.505 INFO:tasks.rados.rados.0.smithi033.stdout:2156: writing smithi03385152-10 from 2465792 to 3211264 tid 3 2024-09-21T09:58:04.507 INFO:tasks.rados.rados.0.smithi033.stdout:2156: writing smithi03385152-10 from 3211264 to 3489792 tid 4 2024-09-21T09:58:04.507 INFO:tasks.rados.rados.0.smithi033.stdout:2157: snap_create 2024-09-21T09:58:04.507 INFO:tasks.rados.rados.0.smithi033.stdout:2155: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:58:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:2156: finishing write tid 1 to smithi03385152-10 2024-09-21T09:58:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:2156: finishing write tid 2 to smithi03385152-10 2024-09-21T09:58:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:2156: finishing write tid 3 to smithi03385152-10 2024-09-21T09:58:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:2156: finishing write tid 4 to smithi03385152-10 2024-09-21T09:58:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:2156: finishing write tid 5 to smithi03385152-10 2024-09-21T09:58:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:2156: finishing write tid 6 to smithi03385152-10 2024-09-21T09:58:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1022 (ObjNum 702 snap 195 seq_num 702) dirty exists 2024-09-21T09:58:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:2156: left oid 10 (ObjNum 702 snap 195 seq_num 702) 2024-09-21T09:58:05.483 INFO:tasks.rados.rados.0.smithi033.stdout:2155: done (2 left) 2024-09-21T09:58:05.483 INFO:tasks.rados.rados.0.smithi033.stdout:2156: done (1 left) 2024-09-21T09:58:05.483 INFO:tasks.rados.rados.0.smithi033.stdout:2157: done (0 left) 2024-09-21T09:58:05.484 INFO:tasks.rados.rados.0.smithi033.stdout:2158: copy_from oid 45 from oid 2 current snap is 196 2024-09-21T09:58:05.484 INFO:tasks.rados.rados.0.smithi033.stdout:2159: read oid 19 snap 173 2024-09-21T09:58:05.484 INFO:tasks.rados.rados.0.smithi033.stdout:2159: expect (ObjNum 600 snap 161 seq_num 600) 2024-09-21T09:58:05.484 INFO:tasks.rados.rados.0.smithi033.stdout:2160: setattr oid 40 current snap is 196 2024-09-21T09:58:05.484 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 196 2024-09-21T09:58:05.484 INFO:tasks.rados.rados.0.smithi033.stdout:2161: seq_num 703 ranges {0=581632,581632=458752,1040384=557056,1597440=417792,2015232=172032} 2024-09-21T09:58:05.488 INFO:tasks.rados.rados.0.smithi033.stdout:2161: writing smithi03385152-49 from 0 to 581632 tid 1 2024-09-21T09:58:05.491 INFO:tasks.rados.rados.0.smithi033.stdout:2161: writing smithi03385152-49 from 581632 to 1040384 tid 2 2024-09-21T09:58:05.495 INFO:tasks.rados.rados.0.smithi033.stdout:2161: writing smithi03385152-49 from 1040384 to 1597440 tid 3 2024-09-21T09:58:05.498 INFO:tasks.rados.rados.0.smithi033.stdout:2161: writing smithi03385152-49 from 1597440 to 2015232 tid 4 2024-09-21T09:58:05.499 INFO:tasks.rados.rados.0.smithi033.stdout:2161: writing smithi03385152-49 from 2015232 to 2187264 tid 5 2024-09-21T09:58:05.499 INFO:tasks.rados.rados.0.smithi033.stdout:2162: rmattr oid 27 current snap is 196 2024-09-21T09:58:05.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 771 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T09:58:05.499 INFO:tasks.rados.rados.0.smithi033.stdout:2159: expect (ObjNum 600 snap 161 seq_num 600) 2024-09-21T09:58:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:2158: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:58:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:2158: finishing copy_from to smithi03385152-45 2024-09-21T09:58:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 918 (ObjNum 65 snap 0 seq_num 1800560998) dirty exists 2024-09-21T09:58:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:2161: finishing write tid 1 to smithi03385152-49 2024-09-21T09:58:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:2161: finishing write tid 2 to smithi03385152-49 2024-09-21T09:58:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:2161: finishing write tid 3 to smithi03385152-49 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1056 (ObjNum 697 snap 193 seq_num 697) dirty exists 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2161: finishing write tid 4 to smithi03385152-49 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2161: finishing write tid 5 to smithi03385152-49 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2161: finishing write tid 6 to smithi03385152-49 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2161: finishing write tid 7 to smithi03385152-49 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1108 (ObjNum 703 snap 196 seq_num 703) dirty exists 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2161: left oid 49 (ObjNum 703 snap 196 seq_num 703) 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2158: done (4 left) 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2159: done (3 left) 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2160: done (2 left) 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2161: done (1 left) 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2162: done (0 left) 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2163: rmattr oid 6 current snap is 196 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2163: done (0 left) 2024-09-21T09:58:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2164: delete oid 1 current snap is 196 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:2164: done (0 left) 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:2165: rmattr oid 5 current snap is 196 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:2165: done (0 left) 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:2166: read oid 16 snap 187 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:2166: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:2167: rollback oid 33 current snap is 196 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 189 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:2168: setattr oid 13 current snap is 196 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:2167: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:58:05.515 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 0 (ObjNum 65 snap 0 seq_num 90749216) dirty dne 2024-09-21T09:58:05.516 INFO:tasks.rados.rados.0.smithi033.stdout:2167: done (2 left) 2024-09-21T09:58:05.516 INFO:tasks.rados.rados.0.smithi033.stdout:2169: read oid 30 snap -1 2024-09-21T09:58:05.516 INFO:tasks.rados.rados.0.smithi033.stdout:2169: expect deleted 2024-09-21T09:58:05.516 INFO:tasks.rados.rados.0.smithi033.stdout:2170: snap_remove snap 193 2024-09-21T09:58:05.715 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-mon[86649]: pgmap v314: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.4 MiB/s wr, 20 op/s; 67/453 objects degraded (14.790%) 2024-09-21T09:58:05.715 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-mon[86649]: osdmap e457: 8 total, 7 up, 8 in 2024-09-21T09:58:05.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.716 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T09:58:05.531+0000 7ff784a39640 -1 osd.4 446 log_to_monitors true 2024-09-21T09:58:05.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[96568]: pgmap v314: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.4 MiB/s wr, 20 op/s; 67/453 objects degraded (14.790%) 2024-09-21T09:58:05.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[96568]: osdmap e457: 8 total, 7 up, 8 in 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[98546]: pgmap v314: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 225 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.4 MiB/s wr, 20 op/s; 67/453 objects degraded (14.790%) 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[98546]: osdmap e457: 8 total, 7 up, 8 in 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:05.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:05 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:06.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:05.740Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:06.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:05 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:05.740Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:06.494 INFO:tasks.rados.rados.0.smithi033.stdout:2170: done (3 left) 2024-09-21T09:58:06.494 INFO:tasks.rados.rados.0.smithi033.stdout:2171: read oid 41 snap -1 2024-09-21T09:58:06.494 INFO:tasks.rados.rados.0.smithi033.stdout:2171: expect (ObjNum 692 snap 192 seq_num 692) 2024-09-21T09:58:06.494 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 196 2024-09-21T09:58:06.494 INFO:tasks.rados.rados.0.smithi033.stdout:2172: seq_num 705 ranges {0=614400,614400=630784,1245184=524288,1769472=450560} 2024-09-21T09:58:06.498 INFO:tasks.rados.rados.0.smithi033.stdout:2172: writing smithi03385152-8 from 0 to 614400 tid 1 2024-09-21T09:58:06.502 INFO:tasks.rados.rados.0.smithi033.stdout:2172: writing smithi03385152-8 from 614400 to 1245184 tid 2 2024-09-21T09:58:06.506 INFO:tasks.rados.rados.0.smithi033.stdout:2172: writing smithi03385152-8 from 1245184 to 1769472 tid 3 2024-09-21T09:58:06.509 INFO:tasks.rados.rados.0.smithi033.stdout:2172: writing smithi03385152-8 from 1769472 to 2220032 tid 4 2024-09-21T09:58:06.509 INFO:tasks.rados.rados.0.smithi033.stdout:2173: delete oid 6 current snap is 196 2024-09-21T09:58:06.509 INFO:tasks.rados.rados.0.smithi033.stdout:2166: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:58:06.526 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:06 smithi105 ceph-mon[86649]: osdmap e458: 8 total, 7 up, 8 in 2024-09-21T09:58:06.526 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:06 smithi105 ceph-mon[86649]: from='osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:58:06.527 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:06 smithi105 ceph-mon[86649]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 1421 (ObjNum 665 snap 185 seq_num 665) dirty exists 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:2172: finishing write tid 1 to smithi03385152-8 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:2172: finishing write tid 2 to smithi03385152-8 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:2166: done (5 left) 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:2168: done (4 left) 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:2169: done (3 left) 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:2173: done (2 left) 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 196 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:2172: finishing write tid 3 to smithi03385152-8 2024-09-21T09:58:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:2172: finishing write tid 4 to smithi03385152-8 2024-09-21T09:58:06.573 INFO:tasks.rados.rados.0.smithi033.stdout:2172: finishing write tid 5 to smithi03385152-8 2024-09-21T09:58:06.573 INFO:tasks.rados.rados.0.smithi033.stdout:2174: seq_num 707 ranges {2187264=647168,2834432=802816,3637248=532480,4169728=270336} 2024-09-21T09:58:06.587 INFO:tasks.rados.rados.0.smithi033.stdout:2174: writing smithi03385152-49 from 2187264 to 2834432 tid 1 2024-09-21T09:58:06.591 INFO:tasks.rados.rados.0.smithi033.stdout:2174: writing smithi03385152-49 from 2834432 to 3637248 tid 2 2024-09-21T09:58:06.595 INFO:tasks.rados.rados.0.smithi033.stdout:2174: writing smithi03385152-49 from 3637248 to 4169728 tid 3 2024-09-21T09:58:06.596 INFO:tasks.rados.rados.0.smithi033.stdout:2174: writing smithi03385152-49 from 4169728 to 4440064 tid 4 2024-09-21T09:58:06.596 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 196 2024-09-21T09:58:06.597 INFO:tasks.rados.rados.0.smithi033.stdout:2171: expect (ObjNum 692 snap 192 seq_num 692) 2024-09-21T09:58:06.705 INFO:tasks.rados.rados.0.smithi033.stdout:2172: finishing write tid 6 to smithi03385152-8 2024-09-21T09:58:06.705 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 917 (ObjNum 705 snap 196 seq_num 705) dirty exists 2024-09-21T09:58:06.705 INFO:tasks.rados.rados.0.smithi033.stdout:2172: left oid 8 (ObjNum 705 snap 196 seq_num 705) 2024-09-21T09:58:06.705 INFO:tasks.rados.rados.0.smithi033.stdout:2175: seq_num 708 ranges {0=663552,663552=737280,1400832=696320,2097152=172032} 2024-09-21T09:58:06.708 INFO:tasks.rados.rados.0.smithi033.stdout:2175: writing smithi03385152-33 from 0 to 663552 tid 1 2024-09-21T09:58:06.712 INFO:tasks.rados.rados.0.smithi033.stdout:2175: writing smithi03385152-33 from 663552 to 1400832 tid 2 2024-09-21T09:58:06.716 INFO:tasks.rados.rados.0.smithi033.stdout:2175: writing smithi03385152-33 from 1400832 to 2097152 tid 3 2024-09-21T09:58:06.717 INFO:tasks.rados.rados.0.smithi033.stdout:2175: writing smithi03385152-33 from 2097152 to 2269184 tid 4 2024-09-21T09:58:06.717 INFO:tasks.rados.rados.0.smithi033.stdout:2171: done (3 left) 2024-09-21T09:58:06.717 INFO:tasks.rados.rados.0.smithi033.stdout:2172: done (2 left) 2024-09-21T09:58:06.718 INFO:tasks.rados.rados.0.smithi033.stdout:2176: snap_remove snap 189 2024-09-21T09:58:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:06 smithi033 ceph-mon[98546]: osdmap e458: 8 total, 7 up, 8 in 2024-09-21T09:58:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:06 smithi033 ceph-mon[98546]: from='osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:58:06.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:06 smithi033 ceph-mon[98546]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:58:06.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:06 smithi033 ceph-mon[96568]: osdmap e458: 8 total, 7 up, 8 in 2024-09-21T09:58:06.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:06 smithi033 ceph-mon[96568]: from='osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:58:06.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:06 smithi033 ceph-mon[96568]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-21T09:58:06.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 09:58:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T09:58:06.524+0000 7ff77c7e3640 -1 osd.4 446 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:58:07.122 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:06.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:07.492 INFO:tasks.rados.rados.0.smithi033.stdout:2176: done (2 left) 2024-09-21T09:58:07.492 INFO:tasks.rados.rados.0.smithi033.stdout:2177: setattr oid 38 current snap is 196 2024-09-21T09:58:07.492 INFO:tasks.rados.rados.0.smithi033.stdout:2174: finishing write tid 1 to smithi03385152-49 2024-09-21T09:58:07.492 INFO:tasks.rados.rados.0.smithi033.stdout:2174: finishing write tid 2 to smithi03385152-49 2024-09-21T09:58:07.492 INFO:tasks.rados.rados.0.smithi033.stdout:2174: finishing write tid 3 to smithi03385152-49 2024-09-21T09:58:07.492 INFO:tasks.rados.rados.0.smithi033.stdout:2174: finishing write tid 4 to smithi03385152-49 2024-09-21T09:58:07.492 INFO:tasks.rados.rados.0.smithi033.stdout:2174: finishing write tid 5 to smithi03385152-49 2024-09-21T09:58:07.492 INFO:tasks.rados.rados.0.smithi033.stdout:2174: finishing write tid 6 to smithi03385152-49 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1113 (ObjNum 707 snap 196 seq_num 707) dirty exists 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:2174: left oid 49 (ObjNum 707 snap 196 seq_num 707) 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:2175: finishing write tid 1 to smithi03385152-33 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:2175: finishing write tid 2 to smithi03385152-33 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:2175: finishing write tid 3 to smithi03385152-33 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:2175: finishing write tid 4 to smithi03385152-33 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:2175: finishing write tid 5 to smithi03385152-33 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:2175: finishing write tid 6 to smithi03385152-33 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1061 (ObjNum 708 snap 196 seq_num 708) dirty exists 2024-09-21T09:58:07.493 INFO:tasks.rados.rados.0.smithi033.stdout:2175: left oid 33 (ObjNum 708 snap 196 seq_num 708) 2024-09-21T09:58:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:2174: done (2 left) 2024-09-21T09:58:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:2175: done (1 left) 2024-09-21T09:58:07.494 INFO:tasks.rados.rados.0.smithi033.stdout:2178: snap_create 2024-09-21T09:58:07.515 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 772 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T09:58:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[98546]: pgmap v317: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.5 MiB/s wr, 23 op/s; 67/474 objects degraded (14.135%) 2024-09-21T09:58:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[98546]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:58:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[98546]: from='osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[98546]: osdmap e459: 8 total, 7 up, 8 in 2024-09-21T09:58:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[98546]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:07.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[96568]: pgmap v317: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.5 MiB/s wr, 23 op/s; 67/474 objects degraded (14.135%) 2024-09-21T09:58:07.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[96568]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:58:07.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[96568]: from='osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:07.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[96568]: osdmap e459: 8 total, 7 up, 8 in 2024-09-21T09:58:07.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:07 smithi033 ceph-mon[96568]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:07.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:07 smithi105 ceph-mon[86649]: pgmap v317: 121 pgs: 32 active+undersized, 14 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.5 MiB/s wr, 23 op/s; 67/474 objects degraded (14.135%) 2024-09-21T09:58:07.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:07 smithi105 ceph-mon[86649]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-21T09:58:07.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:07 smithi105 ceph-mon[86649]: from='osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:07.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:07 smithi105 ceph-mon[86649]: osdmap e459: 8 total, 7 up, 8 in 2024-09-21T09:58:07.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:07 smithi105 ceph-mon[86649]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:08.502 INFO:tasks.rados.rados.0.smithi033.stdout:2177: done (1 left) 2024-09-21T09:58:08.502 INFO:tasks.rados.rados.0.smithi033.stdout:2178: done (0 left) 2024-09-21T09:58:08.502 INFO:tasks.rados.rados.0.smithi033.stdout:2179: snap_remove snap 179 2024-09-21T09:58:08.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:58:08.323+0000 7fc38e955640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-21T09:58:08.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110] boot 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:58:08.748 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 67/474 objects degraded (14.135%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:08.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:58:08.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110] boot 2024-09-21T09:58:08.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:58:08.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:58:08.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:58:08.749 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:08 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 67/474 objects degraded (14.135%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:58:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: osd.4 [v2:172.21.15.105:6800/2250129110,v1:172.21.15.105:6801/2250129110] boot 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: osdmap e460: 8 total, 8 up, 8 in 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:08.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:58:08.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:08 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 67/474 objects degraded (14.135%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:09.499 INFO:tasks.rados.rados.0.smithi033.stdout:2179: done (0 left) 2024-09-21T09:58:09.499 INFO:tasks.rados.rados.0.smithi033.stdout:2180: read oid 8 snap -1 2024-09-21T09:58:09.499 INFO:tasks.rados.rados.0.smithi033.stdout:2180: expect (ObjNum 705 snap 196 seq_num 705) 2024-09-21T09:58:09.499 INFO:tasks.rados.rados.0.smithi033.stdout:2181: read oid 35 snap -1 2024-09-21T09:58:09.500 INFO:tasks.rados.rados.0.smithi033.stdout:2181: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:58:09.500 INFO:tasks.rados.rados.0.smithi033.stdout:2182: read oid 6 snap -1 2024-09-21T09:58:09.500 INFO:tasks.rados.rados.0.smithi033.stdout:2182: expect deleted 2024-09-21T09:58:09.500 INFO:tasks.rados.rados.0.smithi033.stdout:2183: snap_remove snap 188 2024-09-21T09:58:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:09 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:58:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:09 smithi105 ceph-mon[86649]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-21T09:58:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:09 smithi105 ceph-mon[86649]: pgmap v320: 121 pgs: 5 peering, 28 active+undersized, 13 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 20 op/s; 65/474 objects degraded (13.713%) 2024-09-21T09:58:09.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:09 smithi105 ceph-mon[86649]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:58:09.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:09 smithi105 ceph-mon[86649]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:58:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:58:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[98546]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-21T09:58:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[98546]: pgmap v320: 121 pgs: 5 peering, 28 active+undersized, 13 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 20 op/s; 65/474 objects degraded (13.713%) 2024-09-21T09:58:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[98546]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:58:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[98546]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:58:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:58:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[96568]: Upgrade: unsafe to stop osd(s) at this time (8 PGs are or would become offline) 2024-09-21T09:58:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[96568]: pgmap v320: 121 pgs: 5 peering, 28 active+undersized, 13 active+undersized+degraded, 75 active+clean; 231 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 20 op/s; 65/474 objects degraded (13.713%) 2024-09-21T09:58:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[96568]: osdmap e461: 8 total, 8 up, 8 in 2024-09-21T09:58:09.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:09 smithi033 ceph-mon[96568]: osdmap e462: 8 total, 8 up, 8 in 2024-09-21T09:58:10.499 INFO:tasks.rados.rados.0.smithi033.stdout:2183: done (3 left) 2024-09-21T09:58:10.499 INFO:tasks.rados.rados.0.smithi033.stdout:2184: snap_create 2024-09-21T09:58:10.499 INFO:tasks.rados.rados.0.smithi033.stdout:2181: expect (ObjNum 574 snap 155 seq_num 574) 2024-09-21T09:58:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:10.584 INFO:tasks.rados.rados.0.smithi033.stdout:2180: expect (ObjNum 705 snap 196 seq_num 705) 2024-09-21T09:58:11.504 INFO:tasks.rados.rados.0.smithi033.stdout:2180: done (3 left) 2024-09-21T09:58:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:2181: done (2 left) 2024-09-21T09:58:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:2182: done (1 left) 2024-09-21T09:58:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:2184: done (0 left) 2024-09-21T09:58:11.505 INFO:tasks.rados.rados.0.smithi033.stdout:2185: snap_remove snap 195 2024-09-21T09:58:11.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:11 smithi033 ceph-mon[96568]: pgmap v323: 121 pgs: 5 peering, 1 active+recovery_wait+degraded, 25 active+undersized, 8 active+undersized+degraded, 82 active+clean; 233 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 530 KiB/s wr, 13 op/s; 45/474 objects degraded (9.494%); 4.5 MiB/s, 0 objects/s recovering 2024-09-21T09:58:11.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:11 smithi033 ceph-mon[96568]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:58:11.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:11 smithi033 ceph-mon[98546]: pgmap v323: 121 pgs: 5 peering, 1 active+recovery_wait+degraded, 25 active+undersized, 8 active+undersized+degraded, 82 active+clean; 233 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 530 KiB/s wr, 13 op/s; 45/474 objects degraded (9.494%); 4.5 MiB/s, 0 objects/s recovering 2024-09-21T09:58:11.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:11 smithi033 ceph-mon[98546]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:58:11.747 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:58:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:58:11] "GET /metrics HTTP/1.1" 200 36317 "" "Prometheus/2.51.0" 2024-09-21T09:58:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:11 smithi105 ceph-mon[86649]: pgmap v323: 121 pgs: 5 peering, 1 active+recovery_wait+degraded, 25 active+undersized, 8 active+undersized+degraded, 82 active+clean; 233 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 530 KiB/s wr, 13 op/s; 45/474 objects degraded (9.494%); 4.5 MiB/s, 0 objects/s recovering 2024-09-21T09:58:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:11 smithi105 ceph-mon[86649]: osdmap e463: 8 total, 8 up, 8 in 2024-09-21T09:58:12.512 INFO:tasks.rados.rados.0.smithi033.stdout:2185: done (0 left) 2024-09-21T09:58:12.512 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 198 2024-09-21T09:58:12.512 INFO:tasks.rados.rados.0.smithi033.stdout:2186: seq_num 709 ranges {0=679936,679936=614400,1294336=499712,1794048=491520} 2024-09-21T09:58:12.517 INFO:tasks.rados.rados.0.smithi033.stdout:2186: writing smithi03385152-6 from 0 to 679936 tid 1 2024-09-21T09:58:12.521 INFO:tasks.rados.rados.0.smithi033.stdout:2186: writing smithi03385152-6 from 679936 to 1294336 tid 2 2024-09-21T09:58:12.523 INFO:tasks.rados.rados.0.smithi033.stdout:2186: writing smithi03385152-6 from 1294336 to 1794048 tid 3 2024-09-21T09:58:12.527 INFO:tasks.rados.rados.0.smithi033.stdout:2186: writing smithi03385152-6 from 1794048 to 2285568 tid 4 2024-09-21T09:58:12.527 INFO:tasks.rados.rados.0.smithi033.stdout:2187: copy_from oid 20 from oid 45 current snap is 198 2024-09-21T09:58:12.527 INFO:tasks.rados.rados.0.smithi033.stdout:2186: finishing write tid 1 to smithi03385152-6 2024-09-21T09:58:12.527 INFO:tasks.rados.rados.0.smithi033.stdout:2186: finishing write tid 2 to smithi03385152-6 2024-09-21T09:58:12.527 INFO:tasks.rados.rados.0.smithi033.stdout:2186: finishing write tid 3 to smithi03385152-6 2024-09-21T09:58:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:2188: rollback oid 39 current snap is 198 2024-09-21T09:58:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 39 to 196 2024-09-21T09:58:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:2189: read oid 30 snap 192 2024-09-21T09:58:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:2189: expect deleted 2024-09-21T09:58:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:2190: read oid 26 snap 194 2024-09-21T09:58:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:2190: expect (ObjNum 549 snap 148 seq_num 549) 2024-09-21T09:58:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 198 2024-09-21T09:58:12.528 INFO:tasks.rados.rados.0.smithi033.stdout:2191: seq_num 710 ranges {1867776=696320,2564096=499712,3063808=712704,3776512=393216} 2024-09-21T09:58:12.541 INFO:tasks.rados.rados.0.smithi033.stdout:2191: writing smithi03385152-32 from 1867776 to 2564096 tid 1 2024-09-21T09:58:12.545 INFO:tasks.rados.rados.0.smithi033.stdout:2191: writing smithi03385152-32 from 2564096 to 3063808 tid 2 2024-09-21T09:58:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2191: writing smithi03385152-32 from 3063808 to 3776512 tid 3 2024-09-21T09:58:12.552 INFO:tasks.rados.rados.0.smithi033.stdout:2191: writing smithi03385152-32 from 3776512 to 4169728 tid 4 2024-09-21T09:58:12.552 INFO:tasks.rados.rados.0.smithi033.stdout:2192: read oid 3 snap -1 2024-09-21T09:58:12.552 INFO:tasks.rados.rados.0.smithi033.stdout:2192: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2193: snap_create 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2187: finishing copy_from racing read to smithi03385152-20 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2188: finishing rollback tid 0 to smithi03385152-39 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2187: finishing copy_from to smithi03385152-20 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 918 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2186: finishing write tid 4 to smithi03385152-6 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2188: finishing rollback tid 1 to smithi03385152-39 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2186: finishing write tid 5 to smithi03385152-6 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2188: finishing rollback tid 2 to smithi03385152-39 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1009 (ObjNum 622 snap 170 seq_num 622) dirty exists 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:2186: finishing write tid 6 to smithi03385152-6 2024-09-21T09:58:12.553 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1066 (ObjNum 709 snap 198 seq_num 709) dirty exists 2024-09-21T09:58:12.554 INFO:tasks.rados.rados.0.smithi033.stdout:2186: left oid 6 (ObjNum 709 snap 198 seq_num 709) 2024-09-21T09:58:12.554 INFO:tasks.rados.rados.0.smithi033.stdout:2191: finishing write tid 1 to smithi03385152-32 2024-09-21T09:58:12.554 INFO:tasks.rados.rados.0.smithi033.stdout:2191: finishing write tid 2 to smithi03385152-32 2024-09-21T09:58:12.555 INFO:tasks.rados.rados.0.smithi033.stdout:2191: finishing write tid 3 to smithi03385152-32 2024-09-21T09:58:12.555 INFO:tasks.rados.rados.0.smithi033.stdout:2191: finishing write tid 4 to smithi03385152-32 2024-09-21T09:58:12.556 INFO:tasks.rados.rados.0.smithi033.stdout:2191: finishing write tid 5 to smithi03385152-32 2024-09-21T09:58:12.557 INFO:tasks.rados.rados.0.smithi033.stdout:2191: finishing write tid 6 to smithi03385152-32 2024-09-21T09:58:12.557 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 193 (ObjNum 710 snap 198 seq_num 710) dirty exists 2024-09-21T09:58:12.557 INFO:tasks.rados.rados.0.smithi033.stdout:2191: left oid 32 (ObjNum 710 snap 198 seq_num 710) 2024-09-21T09:58:12.557 INFO:tasks.rados.rados.0.smithi033.stdout:2192: expect (ObjNum 450 snap 120 seq_num 450) 2024-09-21T09:58:12.572 INFO:tasks.rados.rados.0.smithi033.stdout:2190: expect (ObjNum 549 snap 148 seq_num 549) 2024-09-21T09:58:12.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:12 smithi105 ceph-mon[86649]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:58:12.927 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:12 smithi033 ceph-mon[98546]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:58:12.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:12 smithi033 ceph-mon[96568]: osdmap e464: 8 total, 8 up, 8 in 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2186: done (7 left) 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2187: done (6 left) 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2188: done (5 left) 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2189: done (4 left) 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2190: done (3 left) 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2191: done (2 left) 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2192: done (1 left) 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2193: done (0 left) 2024-09-21T09:58:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2194: delete oid 37 current snap is 199 2024-09-21T09:58:13.518 INFO:tasks.rados.rados.0.smithi033.stdout:2194: done (0 left) 2024-09-21T09:58:13.519 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 199 2024-09-21T09:58:13.519 INFO:tasks.rados.rados.0.smithi033.stdout:2195: seq_num 712 ranges {3047424=729088,3776512=671744,4448256=671744,5120000=262144} 2024-09-21T09:58:13.541 INFO:tasks.rados.rados.0.smithi033.stdout:2195: writing smithi03385152-4 from 3047424 to 3776512 tid 1 2024-09-21T09:58:13.547 INFO:tasks.rados.rados.0.smithi033.stdout:2195: writing smithi03385152-4 from 3776512 to 4448256 tid 2 2024-09-21T09:58:13.551 INFO:tasks.rados.rados.0.smithi033.stdout:2195: writing smithi03385152-4 from 4448256 to 5120000 tid 3 2024-09-21T09:58:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2195: writing smithi03385152-4 from 5120000 to 5382144 tid 4 2024-09-21T09:58:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2196: snap_create 2024-09-21T09:58:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2195: finishing write tid 1 to smithi03385152-4 2024-09-21T09:58:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2195: finishing write tid 2 to smithi03385152-4 2024-09-21T09:58:13.555 INFO:tasks.rados.rados.0.smithi033.stdout:2195: finishing write tid 3 to smithi03385152-4 2024-09-21T09:58:13.555 INFO:tasks.rados.rados.0.smithi033.stdout:2195: finishing write tid 4 to smithi03385152-4 2024-09-21T09:58:13.555 INFO:tasks.rados.rados.0.smithi033.stdout:2195: finishing write tid 5 to smithi03385152-4 2024-09-21T09:58:13.559 INFO:tasks.rados.rados.0.smithi033.stdout:2195: finishing write tid 6 to smithi03385152-4 2024-09-21T09:58:13.559 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1036 (ObjNum 712 snap 199 seq_num 712) dirty exists 2024-09-21T09:58:13.559 INFO:tasks.rados.rados.0.smithi033.stdout:2195: left oid 4 (ObjNum 712 snap 199 seq_num 712) 2024-09-21T09:58:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:13 smithi105 ceph-mon[86649]: pgmap v326: 121 pgs: 5 peering, 1 active+recovery_wait+degraded, 115 active+clean; 238 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.1 MiB/s wr, 19 op/s; 3/465 objects degraded (0.645%); 13 MiB/s, 3 objects/s recovering 2024-09-21T09:58:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:13 smithi105 ceph-mon[86649]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:58:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:13 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 3/465 objects degraded (0.645%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:58:13.940 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:13 smithi033 ceph-mon[98546]: pgmap v326: 121 pgs: 5 peering, 1 active+recovery_wait+degraded, 115 active+clean; 238 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.1 MiB/s wr, 19 op/s; 3/465 objects degraded (0.645%); 13 MiB/s, 3 objects/s recovering 2024-09-21T09:58:13.940 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:13 smithi033 ceph-mon[98546]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:58:13.940 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:13 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 3/465 objects degraded (0.645%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:58:13.940 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:13 smithi033 ceph-mon[96568]: pgmap v326: 121 pgs: 5 peering, 1 active+recovery_wait+degraded, 115 active+clean; 238 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 2.1 MiB/s wr, 19 op/s; 3/465 objects degraded (0.645%); 13 MiB/s, 3 objects/s recovering 2024-09-21T09:58:13.940 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:13 smithi033 ceph-mon[96568]: osdmap e465: 8 total, 8 up, 8 in 2024-09-21T09:58:13.941 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:13 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 3/465 objects degraded (0.645%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:58:14.519 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:14 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:14.147Z caller=alerting.go:391 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-21T09:58:14.519 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:14 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:14.148Z caller=group.go:483 level=warn name=CephOSDFlapping index=13 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.4\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.4\", ceph_version=\"ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)\", cluster_addr=\"172.21.15.105\", device_class=\"ssd\", hostname=\"smithi105\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.105\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:14.524 INFO:tasks.rados.rados.0.smithi033.stdout:2195: done (1 left) 2024-09-21T09:58:14.524 INFO:tasks.rados.rados.0.smithi033.stdout:2196: done (0 left) 2024-09-21T09:58:14.524 INFO:tasks.rados.rados.0.smithi033.stdout:2197: read oid 39 snap 197 2024-09-21T09:58:14.524 INFO:tasks.rados.rados.0.smithi033.stdout:2197: expect (ObjNum 622 snap 170 seq_num 622) 2024-09-21T09:58:14.524 INFO:tasks.rados.rados.0.smithi033.stdout:2198: snap_remove snap 186 2024-09-21T09:58:14.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:14 smithi105 ceph-mon[86649]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:58:14.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:14 smithi033 ceph-mon[96568]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:58:14.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:14 smithi033 ceph-mon[98546]: osdmap e466: 8 total, 8 up, 8 in 2024-09-21T09:58:15.531 INFO:tasks.rados.rados.0.smithi033.stdout:2198: done (1 left) 2024-09-21T09:58:15.531 INFO:tasks.rados.rados.0.smithi033.stdout:2199: read oid 22 snap -1 2024-09-21T09:58:15.531 INFO:tasks.rados.rados.0.smithi033.stdout:2199: expect deleted 2024-09-21T09:58:15.531 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 200 2024-09-21T09:58:15.531 INFO:tasks.rados.rados.0.smithi033.stdout:2200: seq_num 713 ranges {13508608=745472,14254080=557056,14811136=483328,15294464=565248} 2024-09-21T09:58:15.610 INFO:tasks.rados.rados.0.smithi033.stdout:2200: writing smithi03385152-29 from 13508608 to 14254080 tid 1 2024-09-21T09:58:15.613 INFO:tasks.rados.rados.0.smithi033.stdout:2200: writing smithi03385152-29 from 14254080 to 14811136 tid 2 2024-09-21T09:58:15.616 INFO:tasks.rados.rados.0.smithi033.stdout:2200: writing smithi03385152-29 from 14811136 to 15294464 tid 3 2024-09-21T09:58:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:2200: writing smithi03385152-29 from 15294464 to 15859712 tid 4 2024-09-21T09:58:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:2201: rollback oid 33 current snap is 200 2024-09-21T09:58:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 185 2024-09-21T09:58:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:2197: expect (ObjNum 622 snap 170 seq_num 622) 2024-09-21T09:58:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:2200: finishing write tid 1 to smithi03385152-29 2024-09-21T09:58:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:2200: finishing write tid 2 to smithi03385152-29 2024-09-21T09:58:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:2200: finishing write tid 3 to smithi03385152-29 2024-09-21T09:58:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:2201: finishing rollback tid 0 to smithi03385152-33 2024-09-21T09:58:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:2201: finishing rollback tid 1 to smithi03385152-33 2024-09-21T09:58:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1068 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:58:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:2200: finishing write tid 4 to smithi03385152-29 2024-09-21T09:58:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:2200: finishing write tid 5 to smithi03385152-29 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2200: finishing write tid 6 to smithi03385152-29 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1469 (ObjNum 713 snap 200 seq_num 713) dirty exists 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2200: left oid 29 (ObjNum 713 snap 200 seq_num 713) 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2197: done (3 left) 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2199: done (2 left) 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2200: done (1 left) 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2201: done (0 left) 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2202: rollback oid 17 current snap is 200 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 199 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2203: rollback oid 9 current snap is 200 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 9 to 173 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 200 2024-09-21T09:58:15.644 INFO:tasks.rados.rados.0.smithi033.stdout:2204: seq_num 714 ranges {6496256=761856,7258112=442368,7700480=688128,8388608=475136} 2024-09-21T09:58:15.682 INFO:tasks.rados.rados.0.smithi033.stdout:2204: writing smithi03385152-31 from 6496256 to 7258112 tid 1 2024-09-21T09:58:15.684 INFO:tasks.rados.rados.0.smithi033.stdout:2204: writing smithi03385152-31 from 7258112 to 7700480 tid 2 2024-09-21T09:58:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2204: writing smithi03385152-31 from 7700480 to 8388608 tid 3 2024-09-21T09:58:15.692 INFO:tasks.rados.rados.0.smithi033.stdout:2204: writing smithi03385152-31 from 8388608 to 8863744 tid 4 2024-09-21T09:58:15.692 INFO:tasks.rados.rados.0.smithi033.stdout:2205: read oid 7 snap -1 2024-09-21T09:58:15.692 INFO:tasks.rados.rados.0.smithi033.stdout:2205: expect deleted 2024-09-21T09:58:15.692 INFO:tasks.rados.rados.0.smithi033.stdout:2206: read oid 44 snap 198 2024-09-21T09:58:15.692 INFO:tasks.rados.rados.0.smithi033.stdout:2206: expect (ObjNum 608 snap 162 seq_num 608) 2024-09-21T09:58:15.692 INFO:tasks.rados.rados.0.smithi033.stdout:2207: snap_create 2024-09-21T09:58:15.693 INFO:tasks.rados.rados.0.smithi033.stdout:2202: finishing rollback tid 1 to smithi03385152-17 2024-09-21T09:58:15.693 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 1099 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:58:15.693 INFO:tasks.rados.rados.0.smithi033.stdout:2203: finishing rollback tid 0 to smithi03385152-9 2024-09-21T09:58:15.693 INFO:tasks.rados.rados.0.smithi033.stdout:2203: finishing rollback tid 1 to smithi03385152-9 2024-09-21T09:58:15.693 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 774 (ObjNum 129 snap 0 seq_num 90694920) dirty dne 2024-09-21T09:58:15.693 INFO:tasks.rados.rados.0.smithi033.stdout:2204: finishing write tid 1 to smithi03385152-31 2024-09-21T09:58:15.693 INFO:tasks.rados.rados.0.smithi033.stdout:2204: finishing write tid 2 to smithi03385152-31 2024-09-21T09:58:15.694 INFO:tasks.rados.rados.0.smithi033.stdout:2204: finishing write tid 3 to smithi03385152-31 2024-09-21T09:58:15.695 INFO:tasks.rados.rados.0.smithi033.stdout:2204: finishing write tid 4 to smithi03385152-31 2024-09-21T09:58:15.695 INFO:tasks.rados.rados.0.smithi033.stdout:2204: finishing write tid 5 to smithi03385152-31 2024-09-21T09:58:15.696 INFO:tasks.rados.rados.0.smithi033.stdout:2204: finishing write tid 6 to smithi03385152-31 2024-09-21T09:58:15.697 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1475 (ObjNum 714 snap 200 seq_num 714) dirty exists 2024-09-21T09:58:15.697 INFO:tasks.rados.rados.0.smithi033.stdout:2204: left oid 31 (ObjNum 714 snap 200 seq_num 714) 2024-09-21T09:58:15.723 INFO:tasks.rados.rados.0.smithi033.stdout:2206: expect (ObjNum 608 snap 162 seq_num 608) 2024-09-21T09:58:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:15 smithi105 ceph-mon[86649]: pgmap v329: 121 pgs: 121 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 8 op/s; 9.5 MiB/s, 3 objects/s recovering 2024-09-21T09:58:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:15 smithi105 ceph-mon[86649]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/465 objects degraded (0.645%), 1 pg degraded) 2024-09-21T09:58:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:15 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T09:58:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:15 smithi105 ceph-mon[86649]: osdmap e467: 8 total, 8 up, 8 in 2024-09-21T09:58:15.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:15.741Z caller=group.go:483 level=warn name=CephPGsInactive index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:15.781 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:15.741Z caller=group.go:483 level=warn name=CephPGsUnclean index=1 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:15 smithi033 ceph-mon[96568]: pgmap v329: 121 pgs: 121 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 8 op/s; 9.5 MiB/s, 3 objects/s recovering 2024-09-21T09:58:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:15 smithi033 ceph-mon[96568]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/465 objects degraded (0.645%), 1 pg degraded) 2024-09-21T09:58:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:15 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T09:58:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:15 smithi033 ceph-mon[96568]: osdmap e467: 8 total, 8 up, 8 in 2024-09-21T09:58:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:15 smithi033 ceph-mon[98546]: pgmap v329: 121 pgs: 121 active+clean; 240 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.2 MiB/s wr, 8 op/s; 9.5 MiB/s, 3 objects/s recovering 2024-09-21T09:58:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:15 smithi033 ceph-mon[98546]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/465 objects degraded (0.645%), 1 pg degraded) 2024-09-21T09:58:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:15 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T09:58:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:15 smithi033 ceph-mon[98546]: osdmap e467: 8 total, 8 up, 8 in 2024-09-21T09:58:16.545 INFO:tasks.rados.rados.0.smithi033.stdout:2202: done (5 left) 2024-09-21T09:58:16.545 INFO:tasks.rados.rados.0.smithi033.stdout:2203: done (4 left) 2024-09-21T09:58:16.545 INFO:tasks.rados.rados.0.smithi033.stdout:2204: done (3 left) 2024-09-21T09:58:16.546 INFO:tasks.rados.rados.0.smithi033.stdout:2205: done (2 left) 2024-09-21T09:58:16.546 INFO:tasks.rados.rados.0.smithi033.stdout:2206: done (1 left) 2024-09-21T09:58:16.546 INFO:tasks.rados.rados.0.smithi033.stdout:2207: done (0 left) 2024-09-21T09:58:16.546 INFO:tasks.rados.rados.0.smithi033.stdout:2208: read oid 15 snap -1 2024-09-21T09:58:16.546 INFO:tasks.rados.rados.0.smithi033.stdout:2208: expect (ObjNum 684 snap 188 seq_num 684) 2024-09-21T09:58:16.546 INFO:tasks.rados.rados.0.smithi033.stdout:2209: rmattr oid 36 current snap is 201 2024-09-21T09:58:16.546 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 201 2024-09-21T09:58:16.546 INFO:tasks.rados.rados.0.smithi033.stdout:2210: seq_num 715 ranges {0=778240,778240=770048,1548288=499712,2048000=335872} 2024-09-21T09:58:16.550 INFO:tasks.rados.rados.0.smithi033.stdout:2210: writing smithi03385152-45 from 0 to 778240 tid 1 2024-09-21T09:58:16.555 INFO:tasks.rados.rados.0.smithi033.stdout:2210: writing smithi03385152-45 from 778240 to 1548288 tid 2 2024-09-21T09:58:16.557 INFO:tasks.rados.rados.0.smithi033.stdout:2210: writing smithi03385152-45 from 1548288 to 2048000 tid 3 2024-09-21T09:58:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:2210: writing smithi03385152-45 from 2048000 to 2383872 tid 4 2024-09-21T09:58:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:2211: delete oid 2 current snap is 201 2024-09-21T09:58:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1477 (ObjNum 694 snap 193 seq_num 694) dirty exists 2024-09-21T09:58:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:2210: finishing write tid 1 to smithi03385152-45 2024-09-21T09:58:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:2210: finishing write tid 2 to smithi03385152-45 2024-09-21T09:58:16.561 INFO:tasks.rados.rados.0.smithi033.stdout:2208: expect (ObjNum 684 snap 188 seq_num 684) 2024-09-21T09:58:16.652 INFO:tasks.rados.rados.0.smithi033.stdout:2210: finishing write tid 3 to smithi03385152-45 2024-09-21T09:58:16.652 INFO:tasks.rados.rados.0.smithi033.stdout:2210: finishing write tid 4 to smithi03385152-45 2024-09-21T09:58:16.652 INFO:tasks.rados.rados.0.smithi033.stdout:2210: finishing write tid 5 to smithi03385152-45 2024-09-21T09:58:16.652 INFO:tasks.rados.rados.0.smithi033.stdout:2210: finishing write tid 6 to smithi03385152-45 2024-09-21T09:58:16.652 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1177 (ObjNum 715 snap 201 seq_num 715) dirty exists 2024-09-21T09:58:16.652 INFO:tasks.rados.rados.0.smithi033.stdout:2210: left oid 45 (ObjNum 715 snap 201 seq_num 715) 2024-09-21T09:58:16.653 INFO:tasks.rados.rados.0.smithi033.stdout:2208: done (3 left) 2024-09-21T09:58:16.653 INFO:tasks.rados.rados.0.smithi033.stdout:2209: done (2 left) 2024-09-21T09:58:16.653 INFO:tasks.rados.rados.0.smithi033.stdout:2210: done (1 left) 2024-09-21T09:58:16.653 INFO:tasks.rados.rados.0.smithi033.stdout:2211: done (0 left) 2024-09-21T09:58:16.653 INFO:tasks.rados.rados.0.smithi033.stdout:2212: setattr oid 40 current snap is 201 2024-09-21T09:58:16.654 INFO:tasks.rados.rados.0.smithi033.stdout:2213: read oid 50 snap -1 2024-09-21T09:58:16.654 INFO:tasks.rados.rados.0.smithi033.stdout:2213: expect deleted 2024-09-21T09:58:16.654 INFO:tasks.rados.rados.0.smithi033.stdout:2214: copy_from oid 6 from oid 18 current snap is 201 2024-09-21T09:58:16.654 INFO:tasks.rados.rados.0.smithi033.stdout:2215: read oid 16 snap -1 2024-09-21T09:58:16.654 INFO:tasks.rados.rados.0.smithi033.stdout:2215: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:58:16.654 INFO:tasks.rados.rados.0.smithi033.stdout:2216: snap_create 2024-09-21T09:58:16.655 INFO:tasks.rados.rados.0.smithi033.stdout:2214: finishing copy_from to smithi03385152-6 2024-09-21T09:58:16.655 INFO:tasks.rados.rados.0.smithi033.stdout:2214: got expected ENOENT (src dne) 2024-09-21T09:58:16.655 INFO:tasks.rados.rados.0.smithi033.stdout:2214: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:58:16.658 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 775 (ObjNum 33 snap 0 seq_num 5) dirty exists 2024-09-21T09:58:16.661 INFO:tasks.rados.rados.0.smithi033.stdout:2215: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:58:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:16 smithi105 ceph-mon[86649]: osdmap e468: 8 total, 8 up, 8 in 2024-09-21T09:58:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:16 smithi033 ceph-mon[98546]: osdmap e468: 8 total, 8 up, 8 in 2024-09-21T09:58:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:16.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:16 smithi033 ceph-mon[96568]: osdmap e468: 8 total, 8 up, 8 in 2024-09-21T09:58:16.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:16.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:17.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:16 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:16.949Z caller=group.go:483 level=warn name=CephNodeDiskspaceWarning index=4 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi105\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}, {__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi105\", job=\"node\", machine=\"x86_64\", nodename=\"smithi105\", release=\"5.14.0-508.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Thu Sep 12 15:49:37 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:17.547 INFO:tasks.rados.rados.0.smithi033.stdout:2212: done (4 left) 2024-09-21T09:58:17.547 INFO:tasks.rados.rados.0.smithi033.stdout:2213: done (3 left) 2024-09-21T09:58:17.547 INFO:tasks.rados.rados.0.smithi033.stdout:2214: done (2 left) 2024-09-21T09:58:17.547 INFO:tasks.rados.rados.0.smithi033.stdout:2215: done (1 left) 2024-09-21T09:58:17.547 INFO:tasks.rados.rados.0.smithi033.stdout:2216: done (0 left) 2024-09-21T09:58:17.547 INFO:tasks.rados.rados.0.smithi033.stdout:2217: snap_create 2024-09-21T09:58:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:17 smithi033 ceph-mon[98546]: pgmap v332: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 12 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-21T09:58:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:17 smithi033 ceph-mon[98546]: osdmap e469: 8 total, 8 up, 8 in 2024-09-21T09:58:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:17 smithi033 ceph-mon[96568]: pgmap v332: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 12 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-21T09:58:17.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:17 smithi033 ceph-mon[96568]: osdmap e469: 8 total, 8 up, 8 in 2024-09-21T09:58:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:17 smithi105 ceph-mon[86649]: pgmap v332: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.8 MiB/s wr, 12 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-21T09:58:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:17 smithi105 ceph-mon[86649]: osdmap e469: 8 total, 8 up, 8 in 2024-09-21T09:58:18.486 INFO:tasks.rados.rados.0.smithi033.stdout:2217: done (0 left) 2024-09-21T09:58:18.486 INFO:tasks.rados.rados.0.smithi033.stdout:2218: delete oid 45 current snap is 203 2024-09-21T09:58:18.489 INFO:tasks.rados.rados.0.smithi033.stdout:2218: done (0 left) 2024-09-21T09:58:18.489 INFO:tasks.rados.rados.0.smithi033.stdout:2219: read oid 40 snap -1 2024-09-21T09:58:18.489 INFO:tasks.rados.rados.0.smithi033.stdout:2219: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:58:18.489 INFO:tasks.rados.rados.0.smithi033.stdout:2220: read oid 22 snap 200 2024-09-21T09:58:18.490 INFO:tasks.rados.rados.0.smithi033.stdout:2220: expect deleted 2024-09-21T09:58:18.490 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 203 2024-09-21T09:58:18.490 INFO:tasks.rados.rados.0.smithi033.stdout:2221: seq_num 718 ranges {0=32768} 2024-09-21T09:58:18.490 INFO:tasks.rados.rados.0.smithi033.stdout:2221: writing smithi03385152-33 from 0 to 32768 tid 1 2024-09-21T09:58:18.490 INFO:tasks.rados.rados.0.smithi033.stdout:2222: snap_create 2024-09-21T09:58:18.493 INFO:tasks.rados.rados.0.smithi033.stdout:2219: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:58:18.497 INFO:tasks.rados.rados.0.smithi033.stdout:2221: finishing write tid 1 to smithi03385152-33 2024-09-21T09:58:18.497 INFO:tasks.rados.rados.0.smithi033.stdout:2221: finishing write tid 2 to smithi03385152-33 2024-09-21T09:58:18.499 INFO:tasks.rados.rados.0.smithi033.stdout:2221: finishing write tid 3 to smithi03385152-33 2024-09-21T09:58:18.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1070 (ObjNum 718 snap 203 seq_num 718) dirty exists 2024-09-21T09:58:18.499 INFO:tasks.rados.rados.0.smithi033.stdout:2221: left oid 33 (ObjNum 718 snap 203 seq_num 718) 2024-09-21T09:58:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:18 smithi033 ceph-mon[98546]: osdmap e470: 8 total, 8 up, 8 in 2024-09-21T09:58:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:18 smithi033 ceph-mon[98546]: osdmap e471: 8 total, 8 up, 8 in 2024-09-21T09:58:18.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:18 smithi033 ceph-mon[96568]: osdmap e470: 8 total, 8 up, 8 in 2024-09-21T09:58:18.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:18 smithi033 ceph-mon[96568]: osdmap e471: 8 total, 8 up, 8 in 2024-09-21T09:58:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:18 smithi105 ceph-mon[86649]: osdmap e470: 8 total, 8 up, 8 in 2024-09-21T09:58:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:18 smithi105 ceph-mon[86649]: osdmap e471: 8 total, 8 up, 8 in 2024-09-21T09:58:19.230 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2219: done (3 left) 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2220: done (2 left) 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2221: done (1 left) 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2222: done (0 left) 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2223: copy_from oid 47 from oid 36 current snap is 204 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 204 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2224: seq_num 719 ranges {0=49152} 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2224: writing smithi03385152-9 from 0 to 49152 tid 1 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2225: read oid 33 snap -1 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2225: expect (ObjNum 718 snap 203 seq_num 718) 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 204 2024-09-21T09:58:19.496 INFO:tasks.rados.rados.0.smithi033.stdout:2226: seq_num 720 ranges {0=65536} 2024-09-21T09:58:19.497 INFO:tasks.rados.rados.0.smithi033.stdout:2226: writing smithi03385152-50 from 0 to 65536 tid 1 2024-09-21T09:58:19.497 INFO:tasks.rados.rados.0.smithi033.stdout:2227: copy_from oid 17 from oid 18 current snap is 204 2024-09-21T09:58:19.497 INFO:tasks.rados.rados.0.smithi033.stdout:2228: read oid 26 snap -1 2024-09-21T09:58:19.497 INFO:tasks.rados.rados.0.smithi033.stdout:2228: expect (ObjNum 549 snap 148 seq_num 549) 2024-09-21T09:58:19.497 INFO:tasks.rados.rados.0.smithi033.stdout:2229: snap_create 2024-09-21T09:58:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:2224: finishing write tid 1 to smithi03385152-9 2024-09-21T09:58:19.500 INFO:tasks.rados.rados.0.smithi033.stdout:2224: finishing write tid 2 to smithi03385152-9 2024-09-21T09:58:19.501 INFO:tasks.rados.rados.0.smithi033.stdout:2226: finishing write tid 1 to smithi03385152-50 2024-09-21T09:58:19.501 INFO:tasks.rados.rados.0.smithi033.stdout:2226: finishing write tid 2 to smithi03385152-50 2024-09-21T09:58:19.502 INFO:tasks.rados.rados.0.smithi033.stdout:2227: finishing copy_from to smithi03385152-17 2024-09-21T09:58:19.502 INFO:tasks.rados.rados.0.smithi033.stdout:2227: got expected ENOENT (src dne) 2024-09-21T09:58:19.502 INFO:tasks.rados.rados.0.smithi033.stdout:2224: finishing write tid 3 to smithi03385152-9 2024-09-21T09:58:19.502 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 777 (ObjNum 719 snap 204 seq_num 719) dirty exists 2024-09-21T09:58:19.502 INFO:tasks.rados.rados.0.smithi033.stdout:2224: left oid 9 (ObjNum 719 snap 204 seq_num 719) 2024-09-21T09:58:19.502 INFO:tasks.rados.rados.0.smithi033.stdout:2227: finishing copy_from racing read to smithi03385152-17 2024-09-21T09:58:19.503 INFO:tasks.rados.rados.0.smithi033.stdout:2226: finishing write tid 3 to smithi03385152-50 2024-09-21T09:58:19.503 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 921 (ObjNum 720 snap 204 seq_num 720) dirty exists 2024-09-21T09:58:19.503 INFO:tasks.rados.rados.0.smithi033.stdout:2226: left oid 50 (ObjNum 720 snap 204 seq_num 720) 2024-09-21T09:58:19.522 INFO:tasks.rados.rados.0.smithi033.stdout:2225: expect (ObjNum 718 snap 203 seq_num 718) 2024-09-21T09:58:19.526 INFO:tasks.rados.rados.0.smithi033.stdout:2223: finishing copy_from racing read to smithi03385152-47 2024-09-21T09:58:19.529 INFO:tasks.rados.rados.0.smithi033.stdout:2223: finishing copy_from to smithi03385152-47 2024-09-21T09:58:19.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1477 (ObjNum 694 snap 193 seq_num 694) dirty exists 2024-09-21T09:58:19.538 INFO:tasks.rados.rados.0.smithi033.stdout:2228: expect (ObjNum 549 snap 148 seq_num 549) 2024-09-21T09:58:19.826 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:58:19.826 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (5m) 49s ago 9m 19.2M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:58:19.826 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (4m) 15s ago 9m 87.5M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:58:19.826 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (11m) 49s ago 11m 16.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:58:19.826 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (11m) 15s ago 11m 21.3M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:58:19.826 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (4m) 15s ago 14m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:58:19.826 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (4m) 49s ago 16m 524M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (3m) 49s ago 16m 56.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (3m) 15s ago 14m 46.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (3m) 49s ago 14m 47.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (5m) 49s ago 9m 11.8M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (4m) 15s ago 9m 11.6M - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (2m) 49s ago 14m 168M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (2m) 49s ago 13m 126M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (87s) 49s ago 13m 66.9M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 76dbf29565ba 2024-09-21T09:58:19.827 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (53s) 49s ago 13m 11.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b85b61817088 2024-09-21T09:58:19.828 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (20s) 15s ago 12m 12.2M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 01f4f66dbe72 2024-09-21T09:58:19.828 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (12m) 15s ago 12m 317M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be c7fc724fff44 2024-09-21T09:58:19.828 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (11m) 15s ago 11m 191M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:58:19.828 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (11m) 15s ago 11m 195M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:58:19.828 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (4m) 15s ago 9m 51.5M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:58:19.828 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:19 smithi033 ceph-mon[98546]: pgmap v335: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T09:58:19.829 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:19 smithi033 ceph-mon[98546]: osdmap e472: 8 total, 8 up, 8 in 2024-09-21T09:58:19.829 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:19 smithi033 ceph-mon[96568]: pgmap v335: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T09:58:19.829 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:19 smithi033 ceph-mon[96568]: osdmap e472: 8 total, 8 up, 8 in 2024-09-21T09:58:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:19 smithi105 ceph-mon[86649]: pgmap v335: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T09:58:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:19 smithi105 ceph-mon[86649]: osdmap e472: 8 total, 8 up, 8 in 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3, 2024-09-21T09:58:20.179 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 5 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 5, 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 10 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:58:20.180 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:58:20.479 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:58:20.479 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "10/20 daemons upgraded", 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:58:20.480 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:58:20.493 INFO:tasks.rados.rados.0.smithi033.stdout:2223: done (6 left) 2024-09-21T09:58:20.493 INFO:tasks.rados.rados.0.smithi033.stdout:2224: done (5 left) 2024-09-21T09:58:20.493 INFO:tasks.rados.rados.0.smithi033.stdout:2225: done (4 left) 2024-09-21T09:58:20.493 INFO:tasks.rados.rados.0.smithi033.stdout:2226: done (3 left) 2024-09-21T09:58:20.493 INFO:tasks.rados.rados.0.smithi033.stdout:2227: done (2 left) 2024-09-21T09:58:20.494 INFO:tasks.rados.rados.0.smithi033.stdout:2228: done (1 left) 2024-09-21T09:58:20.494 INFO:tasks.rados.rados.0.smithi033.stdout:2229: done (0 left) 2024-09-21T09:58:20.494 INFO:tasks.rados.rados.0.smithi033.stdout:2230: delete oid 16 current snap is 205 2024-09-21T09:58:20.496 INFO:tasks.rados.rados.0.smithi033.stdout:2230: done (0 left) 2024-09-21T09:58:20.496 INFO:tasks.rados.rados.0.smithi033.stdout:2231: rollback oid 28 current snap is 205 2024-09-21T09:58:20.496 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 187 2024-09-21T09:58:20.496 INFO:tasks.rados.rados.0.smithi033.stdout:2232: copy_from oid 35 from oid 21 current snap is 205 2024-09-21T09:58:20.496 INFO:tasks.rados.rados.0.smithi033.stdout:2233: delete oid 50 current snap is 205 2024-09-21T09:58:20.499 INFO:tasks.rados.rados.0.smithi033.stdout:2231: finishing rollback tid 0 to smithi03385152-28 2024-09-21T09:58:20.501 INFO:tasks.rados.rados.0.smithi033.stdout:2231: finishing rollback tid 1 to smithi03385152-28 2024-09-21T09:58:20.501 INFO:tasks.rados.rados.0.smithi033.stdout:2233: done (2 left) 2024-09-21T09:58:20.501 INFO:tasks.rados.rados.0.smithi033.stdout:2234: setattr oid 44 current snap is 205 2024-09-21T09:58:20.502 INFO:tasks.rados.rados.0.smithi033.stdout:2231: finishing rollback tid 2 to smithi03385152-28 2024-09-21T09:58:20.502 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 885 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:58:20.503 INFO:tasks.rados.rados.0.smithi033.stdout:2231: done (2 left) 2024-09-21T09:58:20.503 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 205 2024-09-21T09:58:20.503 INFO:tasks.rados.rados.0.smithi033.stdout:2235: seq_num 723 ranges {0=114688} 2024-09-21T09:58:20.504 INFO:tasks.rados.rados.0.smithi033.stdout:2235: writing smithi03385152-1 from 0 to 114688 tid 1 2024-09-21T09:58:20.504 INFO:tasks.rados.rados.0.smithi033.stdout:2236: read oid 12 snap -1 2024-09-21T09:58:20.504 INFO:tasks.rados.rados.0.smithi033.stdout:2236: expect deleted 2024-09-21T09:58:20.504 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 205 2024-09-21T09:58:20.504 INFO:tasks.rados.rados.0.smithi033.stdout:2237: seq_num 724 ranges {4169728=131072} 2024-09-21T09:58:20.528 INFO:tasks.rados.rados.0.smithi033.stdout:2237: writing smithi03385152-32 from 4169728 to 4300800 tid 1 2024-09-21T09:58:20.528 INFO:tasks.rados.rados.0.smithi033.stdout:2238: snap_create 2024-09-21T09:58:20.528 INFO:tasks.rados.rados.0.smithi033.stdout:2235: finishing write tid 1 to smithi03385152-1 2024-09-21T09:58:20.528 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1037 (ObjNum 608 snap 162 seq_num 608) dirty exists 2024-09-21T09:58:20.528 INFO:tasks.rados.rados.0.smithi033.stdout:2235: finishing write tid 2 to smithi03385152-1 2024-09-21T09:58:20.528 INFO:tasks.rados.rados.0.smithi033.stdout:2235: finishing write tid 3 to smithi03385152-1 2024-09-21T09:58:20.528 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 919 (ObjNum 723 snap 205 seq_num 723) dirty exists 2024-09-21T09:58:20.528 INFO:tasks.rados.rados.0.smithi033.stdout:2235: left oid 1 (ObjNum 723 snap 205 seq_num 723) 2024-09-21T09:58:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:20.531 INFO:tasks.rados.rados.0.smithi033.stdout:2237: finishing write tid 1 to smithi03385152-32 2024-09-21T09:58:20.531 INFO:tasks.rados.rados.0.smithi033.stdout:2237: finishing write tid 2 to smithi03385152-32 2024-09-21T09:58:20.532 INFO:tasks.rados.rados.0.smithi033.stdout:2237: finishing write tid 3 to smithi03385152-32 2024-09-21T09:58:20.532 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 198 (ObjNum 724 snap 205 seq_num 724) dirty exists 2024-09-21T09:58:20.532 INFO:tasks.rados.rados.0.smithi033.stdout:2237: left oid 32 (ObjNum 724 snap 205 seq_num 724) 2024-09-21T09:58:20.554 INFO:tasks.rados.rados.0.smithi033.stdout:2232: finishing copy_from racing read to smithi03385152-35 2024-09-21T09:58:20.560 INFO:tasks.rados.rados.0.smithi033.stdout:2232: finishing copy_from to smithi03385152-35 2024-09-21T09:58:20.560 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 1016 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:58:20.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[98546]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:20.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[98546]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:20.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[98546]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:20.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/183094170' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:20.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[98546]: osdmap e473: 8 total, 8 up, 8 in 2024-09-21T09:58:20.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[96568]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:20.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[96568]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:20.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[96568]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:20.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/183094170' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:20.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:20 smithi033 ceph-mon[96568]: osdmap e473: 8 total, 8 up, 8 in 2024-09-21T09:58:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:20 smithi105 ceph-mon[86649]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:20 smithi105 ceph-mon[86649]: from='client.44315 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:20 smithi105 ceph-mon[86649]: from='client.44321 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:20 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/183094170' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:20 smithi105 ceph-mon[86649]: osdmap e473: 8 total, 8 up, 8 in 2024-09-21T09:58:21.495 INFO:tasks.rados.rados.0.smithi033.stdout:2232: done (5 left) 2024-09-21T09:58:21.495 INFO:tasks.rados.rados.0.smithi033.stdout:2234: done (4 left) 2024-09-21T09:58:21.496 INFO:tasks.rados.rados.0.smithi033.stdout:2235: done (3 left) 2024-09-21T09:58:21.496 INFO:tasks.rados.rados.0.smithi033.stdout:2236: done (2 left) 2024-09-21T09:58:21.496 INFO:tasks.rados.rados.0.smithi033.stdout:2237: done (1 left) 2024-09-21T09:58:21.496 INFO:tasks.rados.rados.0.smithi033.stdout:2238: done (0 left) 2024-09-21T09:58:21.496 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 206 2024-09-21T09:58:21.496 INFO:tasks.rados.rados.0.smithi033.stdout:2239: seq_num 725 ranges {2285568=147456} 2024-09-21T09:58:21.508 INFO:tasks.rados.rados.0.smithi033.stdout:2239: writing smithi03385152-6 from 2285568 to 2433024 tid 1 2024-09-21T09:58:21.508 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 206 2024-09-21T09:58:21.509 INFO:tasks.rados.rados.0.smithi033.stdout:2240: seq_num 726 ranges {4308992=163840} 2024-09-21T09:58:21.534 INFO:tasks.rados.rados.0.smithi033.stdout:2240: writing smithi03385152-36 from 4308992 to 4472832 tid 1 2024-09-21T09:58:21.534 INFO:tasks.rados.rados.0.smithi033.stdout:2241: snap_create 2024-09-21T09:58:21.534 INFO:tasks.rados.rados.0.smithi033.stdout:2239: finishing write tid 1 to smithi03385152-6 2024-09-21T09:58:21.534 INFO:tasks.rados.rados.0.smithi033.stdout:2239: finishing write tid 2 to smithi03385152-6 2024-09-21T09:58:21.535 INFO:tasks.rados.rados.0.smithi033.stdout:2239: finishing write tid 3 to smithi03385152-6 2024-09-21T09:58:21.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1072 (ObjNum 725 snap 206 seq_num 725) dirty exists 2024-09-21T09:58:21.535 INFO:tasks.rados.rados.0.smithi033.stdout:2239: left oid 6 (ObjNum 725 snap 206 seq_num 725) 2024-09-21T09:58:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2240: finishing write tid 1 to smithi03385152-36 2024-09-21T09:58:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2240: finishing write tid 2 to smithi03385152-36 2024-09-21T09:58:21.542 INFO:tasks.rados.rados.0.smithi033.stdout:2240: finishing write tid 3 to smithi03385152-36 2024-09-21T09:58:21.542 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1484 (ObjNum 726 snap 206 seq_num 726) dirty exists 2024-09-21T09:58:21.542 INFO:tasks.rados.rados.0.smithi033.stdout:2240: left oid 36 (ObjNum 726 snap 206 seq_num 726) 2024-09-21T09:58:21.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:58:21] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.51.0" 2024-09-21T09:58:21.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-mon[98546]: pgmap v338: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 629 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:58:21.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-mon[98546]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:21.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:21.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-mon[98546]: osdmap e474: 8 total, 8 up, 8 in 2024-09-21T09:58:21.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-mon[96568]: pgmap v338: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 629 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:58:21.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-mon[96568]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:21.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:21.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:21 smithi033 ceph-mon[96568]: osdmap e474: 8 total, 8 up, 8 in 2024-09-21T09:58:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:21 smithi105 ceph-mon[86649]: pgmap v338: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 629 KiB/s rd, 0 B/s wr, 7 op/s 2024-09-21T09:58:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:21 smithi105 ceph-mon[86649]: from='client.34419 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:21 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:21 smithi105 ceph-mon[86649]: osdmap e474: 8 total, 8 up, 8 in 2024-09-21T09:58:22.500 INFO:tasks.rados.rados.0.smithi033.stdout:2239: done (2 left) 2024-09-21T09:58:22.501 INFO:tasks.rados.rados.0.smithi033.stdout:2240: done (1 left) 2024-09-21T09:58:22.501 INFO:tasks.rados.rados.0.smithi033.stdout:2241: done (0 left) 2024-09-21T09:58:22.501 INFO:tasks.rados.rados.0.smithi033.stdout:2242: rmattr oid 11 current snap is 207 2024-09-21T09:58:22.501 INFO:tasks.rados.rados.0.smithi033.stdout:2242: done (0 left) 2024-09-21T09:58:22.501 INFO:tasks.rados.rados.0.smithi033.stdout:2243: read oid 45 snap -1 2024-09-21T09:58:22.501 INFO:tasks.rados.rados.0.smithi033.stdout:2243: expect deleted 2024-09-21T09:58:22.501 INFO:tasks.rados.rados.0.smithi033.stdout:2244: copy_from oid 30 from oid 9 current snap is 207 2024-09-21T09:58:22.501 INFO:tasks.rados.rados.0.smithi033.stdout:2245: snap_create 2024-09-21T09:58:22.506 INFO:tasks.rados.rados.0.smithi033.stdout:2244: finishing copy_from racing read to smithi03385152-30 2024-09-21T09:58:22.508 INFO:tasks.rados.rados.0.smithi033.stdout:2244: finishing copy_from to smithi03385152-30 2024-09-21T09:58:22.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 777 (ObjNum 719 snap 204 seq_num 719) dirty exists 2024-09-21T09:58:23.491 INFO:tasks.rados.rados.0.smithi033.stdout:2243: done (2 left) 2024-09-21T09:58:23.491 INFO:tasks.rados.rados.0.smithi033.stdout:2244: done (1 left) 2024-09-21T09:58:23.491 INFO:tasks.rados.rados.0.smithi033.stdout:2245: done (0 left) 2024-09-21T09:58:23.491 INFO:tasks.rados.rados.0.smithi033.stdout:2246: rollback oid 30 current snap is 208 2024-09-21T09:58:23.491 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 30 to 196 2024-09-21T09:58:23.491 INFO:tasks.rados.rados.0.smithi033.stdout:2247: rollback oid 36 current snap is 208 2024-09-21T09:58:23.492 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 202 2024-09-21T09:58:23.492 INFO:tasks.rados.rados.0.smithi033.stdout:2248: read oid 1 snap -1 2024-09-21T09:58:23.492 INFO:tasks.rados.rados.0.smithi033.stdout:2248: expect (ObjNum 723 snap 205 seq_num 723) 2024-09-21T09:58:23.492 INFO:tasks.rados.rados.0.smithi033.stdout:2249: snap_remove snap 187 2024-09-21T09:58:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:23 smithi033 ceph-mon[98546]: pgmap v341: 121 pgs: 121 active+clean; 265 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-09-21T09:58:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:23 smithi033 ceph-mon[98546]: osdmap e475: 8 total, 8 up, 8 in 2024-09-21T09:58:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:23 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:58:23.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:23 smithi033 ceph-mon[98546]: osdmap e476: 8 total, 8 up, 8 in 2024-09-21T09:58:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:23 smithi033 ceph-mon[96568]: pgmap v341: 121 pgs: 121 active+clean; 265 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-09-21T09:58:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:23 smithi033 ceph-mon[96568]: osdmap e475: 8 total, 8 up, 8 in 2024-09-21T09:58:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:23 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:58:23.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:23 smithi033 ceph-mon[96568]: osdmap e476: 8 total, 8 up, 8 in 2024-09-21T09:58:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:23 smithi105 ceph-mon[86649]: pgmap v341: 121 pgs: 121 active+clean; 265 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-09-21T09:58:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:23 smithi105 ceph-mon[86649]: osdmap e475: 8 total, 8 up, 8 in 2024-09-21T09:58:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:23 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16} : dispatch 2024-09-21T09:58:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:23 smithi105 ceph-mon[86649]: osdmap e476: 8 total, 8 up, 8 in 2024-09-21T09:58:24.493 INFO:tasks.rados.rados.0.smithi033.stdout:2249: done (3 left) 2024-09-21T09:58:24.493 INFO:tasks.rados.rados.0.smithi033.stdout:2250: snap_create 2024-09-21T09:58:24.493 INFO:tasks.rados.rados.0.smithi033.stdout:2248: expect (ObjNum 723 snap 205 seq_num 723) 2024-09-21T09:58:24.499 INFO:tasks.rados.rados.0.smithi033.stdout:2246: finishing rollback tid 0 to smithi03385152-30 2024-09-21T09:58:24.499 INFO:tasks.rados.rados.0.smithi033.stdout:2246: finishing rollback tid 1 to smithi03385152-30 2024-09-21T09:58:24.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 1488 (ObjNum 145 snap 0 seq_num 91377664) dirty dne 2024-09-21T09:58:24.499 INFO:tasks.rados.rados.0.smithi033.stdout:2247: finishing rollback tid 0 to smithi03385152-36 2024-09-21T09:58:24.499 INFO:tasks.rados.rados.0.smithi033.stdout:2247: finishing rollback tid 1 to smithi03385152-36 2024-09-21T09:58:24.499 INFO:tasks.rados.rados.0.smithi033.stdout:2247: finishing rollback tid 2 to smithi03385152-36 2024-09-21T09:58:24.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1492 (ObjNum 694 snap 193 seq_num 694) dirty exists 2024-09-21T09:58:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:24 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:58:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:24 smithi105 ceph-mon[86649]: Upgrade: osd.5 is safe to restart 2024-09-21T09:58:24.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:24 smithi105 ceph-mon[86649]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:58:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:24 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:58:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:24 smithi033 ceph-mon[98546]: Upgrade: osd.5 is safe to restart 2024-09-21T09:58:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:24 smithi033 ceph-mon[98546]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:58:24.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:24 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-09-21T09:58:24.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:24 smithi033 ceph-mon[96568]: Upgrade: osd.5 is safe to restart 2024-09-21T09:58:24.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:24 smithi033 ceph-mon[96568]: osdmap e477: 8 total, 8 up, 8 in 2024-09-21T09:58:25.492 INFO:tasks.rados.rados.0.smithi033.stdout:2246: done (3 left) 2024-09-21T09:58:25.492 INFO:tasks.rados.rados.0.smithi033.stdout:2247: done (2 left) 2024-09-21T09:58:25.492 INFO:tasks.rados.rados.0.smithi033.stdout:2248: done (1 left) 2024-09-21T09:58:25.492 INFO:tasks.rados.rados.0.smithi033.stdout:2250: done (0 left) 2024-09-21T09:58:25.492 INFO:tasks.rados.rados.0.smithi033.stdout:2251: snap_create 2024-09-21T09:58:26.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[98546]: pgmap v344: 121 pgs: 121 active+clean; 269 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 4.9 MiB/s wr, 30 op/s 2024-09-21T09:58:26.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[98546]: Upgrade: Updating osd.5 2024-09-21T09:58:26.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:26.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:58:26.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:26.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[98546]: Deploying daemon osd.5 on smithi105 2024-09-21T09:58:26.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[98546]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:58:26.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[96568]: pgmap v344: 121 pgs: 121 active+clean; 269 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 4.9 MiB/s wr, 30 op/s 2024-09-21T09:58:26.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[96568]: Upgrade: Updating osd.5 2024-09-21T09:58:26.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:26.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:58:26.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:26.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[96568]: Deploying daemon osd.5 on smithi105 2024-09-21T09:58:26.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:25 smithi033 ceph-mon[96568]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:58:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:25 smithi105 ceph-mon[86649]: pgmap v344: 121 pgs: 121 active+clean; 269 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 4.9 MiB/s wr, 30 op/s 2024-09-21T09:58:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:25 smithi105 ceph-mon[86649]: Upgrade: Updating osd.5 2024-09-21T09:58:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:25 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:25 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-21T09:58:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:25 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:25 smithi105 ceph-mon[86649]: Deploying daemon osd.5 on smithi105 2024-09-21T09:58:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:25 smithi105 ceph-mon[86649]: osdmap e478: 8 total, 8 up, 8 in 2024-09-21T09:58:26.494 INFO:tasks.rados.rados.0.smithi033.stdout:2251: done (0 left) 2024-09-21T09:58:26.880 INFO:tasks.rados.rados.0.smithi033.stdout:2252: setattr oid 22 current snap is 210 2024-09-21T09:58:26.881 INFO:tasks.rados.rados.0.smithi033.stdout:2253: snap_remove snap 185 2024-09-21T09:58:27.499 INFO:tasks.rados.rados.0.smithi033.stdout:2253: done (1 left) 2024-09-21T09:58:27.499 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 210 2024-09-21T09:58:27.499 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1184 (ObjNum 33 snap 0 seq_num 1498810389) dirty exists 2024-09-21T09:58:27.499 INFO:tasks.rados.rados.0.smithi033.stdout:2254: seq_num 727 ranges {5947392=180224} 2024-09-21T09:58:27.534 INFO:tasks.rados.rados.0.smithi033.stdout:2254: writing smithi03385152-11 from 5947392 to 6127616 tid 1 2024-09-21T09:58:27.534 INFO:tasks.rados.rados.0.smithi033.stdout:2252: done (1 left) 2024-09-21T09:58:27.534 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 210 2024-09-21T09:58:27.535 INFO:tasks.rados.rados.0.smithi033.stdout:2255: seq_num 728 ranges {6447104=196608} 2024-09-21T09:58:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:2255: writing smithi03385152-44 from 6447104 to 6643712 tid 1 2024-09-21T09:58:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:2256: setattr oid 23 current snap is 210 2024-09-21T09:58:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:2254: finishing write tid 1 to smithi03385152-11 2024-09-21T09:58:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:2254: finishing write tid 2 to smithi03385152-11 2024-09-21T09:58:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:2254: finishing write tid 3 to smithi03385152-11 2024-09-21T09:58:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1029 (ObjNum 727 snap 210 seq_num 727) dirty exists 2024-09-21T09:58:27.571 INFO:tasks.rados.rados.0.smithi033.stdout:2254: left oid 11 (ObjNum 727 snap 210 seq_num 727) 2024-09-21T09:58:27.572 INFO:tasks.rados.rados.0.smithi033.stdout:2254: done (2 left) 2024-09-21T09:58:27.572 INFO:tasks.rados.rados.0.smithi033.stdout:2257: snap_remove snap 203 2024-09-21T09:58:27.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:27 smithi033 ceph-mon[96568]: pgmap v347: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:58:27.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:27 smithi033 ceph-mon[96568]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:58:27.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:27 smithi033 ceph-mon[98546]: pgmap v347: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:58:27.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:27 smithi033 ceph-mon[98546]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:58:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:27 smithi105 ceph-mon[86649]: pgmap v347: 121 pgs: 121 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1.2 MiB/s wr, 4 op/s 2024-09-21T09:58:27.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:27 smithi105 ceph-mon[86649]: osdmap e479: 8 total, 8 up, 8 in 2024-09-21T09:58:28.127 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:27 smithi105 systemd[1]: Stopping Ceph osd.5 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:58:28.502 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[42977]: 2024-09-21T09:58:28.125+0000 7f0117b31640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:58:28.503 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[42977]: 2024-09-21T09:58:28.125+0000 7f0117b31640 -1 osd.5 480 *** Got signal Terminated *** 2024-09-21T09:58:28.503 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[42977]: 2024-09-21T09:58:28.125+0000 7f0117b31640 -1 osd.5 480 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:58:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:2255: finishing write tid 1 to smithi03385152-44 2024-09-21T09:58:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:2255: finishing write tid 2 to smithi03385152-44 2024-09-21T09:58:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:2255: finishing write tid 3 to smithi03385152-44 2024-09-21T09:58:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1039 (ObjNum 728 snap 210 seq_num 728) dirty exists 2024-09-21T09:58:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:2255: left oid 44 (ObjNum 728 snap 210 seq_num 728) 2024-09-21T09:58:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1022 (ObjNum 689 snap 192 seq_num 689) dirty exists 2024-09-21T09:58:28.508 INFO:tasks.rados.rados.0.smithi033.stdout:2255: done (2 left) 2024-09-21T09:58:28.509 INFO:tasks.rados.rados.0.smithi033.stdout:2256: done (1 left) 2024-09-21T09:58:28.509 INFO:tasks.rados.rados.0.smithi033.stdout:2257: done (0 left) 2024-09-21T09:58:28.509 INFO:tasks.rados.rados.0.smithi033.stdout:2258: read oid 24 snap -1 2024-09-21T09:58:28.509 INFO:tasks.rados.rados.0.smithi033.stdout:2258: expect deleted 2024-09-21T09:58:28.509 INFO:tasks.rados.rados.0.smithi033.stdout:2259: delete oid 9 current snap is 210 2024-09-21T09:58:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:2258: done (1 left) 2024-09-21T09:58:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:2259: done (0 left) 2024-09-21T09:58:28.520 INFO:tasks.rados.rados.0.smithi033.stdout:2260: snap_remove snap 200 2024-09-21T09:58:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:28 smithi105 ceph-mon[86649]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:58:28.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:28 smithi105 ceph-mon[86649]: osd.5 marked itself down and dead 2024-09-21T09:58:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:28 smithi033 ceph-mon[98546]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:58:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:28 smithi033 ceph-mon[98546]: osd.5 marked itself down and dead 2024-09-21T09:58:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:28 smithi033 ceph-mon[96568]: osdmap e480: 8 total, 8 up, 8 in 2024-09-21T09:58:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:28 smithi033 ceph-mon[96568]: osd.5 marked itself down and dead 2024-09-21T09:58:29.377 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:29 smithi105 podman[96419]: 2024-09-21 09:58:29.076805325 +0000 UTC m=+1.057459703 container died c7fc724fff44aa8a5d9b89de39a4439de8bd4bbb60524c6c5def2afb14c7b975 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-fbd1a6a, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:58:29.510 INFO:tasks.rados.rados.0.smithi033.stdout:2260: done (0 left) 2024-09-21T09:58:29.510 INFO:tasks.rados.rados.0.smithi033.stdout:2261: rollback oid 11 current snap is 210 2024-09-21T09:58:29.510 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 11 to 199 2024-09-21T09:58:29.510 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 210 2024-09-21T09:58:29.510 INFO:tasks.rados.rados.0.smithi033.stdout:2262: seq_num 730 ranges {2433024=229376} 2024-09-21T09:58:29.526 INFO:tasks.rados.rados.0.smithi033.stdout:2262: writing smithi03385152-6 from 2433024 to 2662400 tid 1 2024-09-21T09:58:29.526 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 210 2024-09-21T09:58:29.526 INFO:tasks.rados.rados.0.smithi033.stdout:2263: seq_num 731 ranges {0=245760} 2024-09-21T09:58:29.528 INFO:tasks.rados.rados.0.smithi033.stdout:2263: writing smithi03385152-37 from 0 to 245760 tid 1 2024-09-21T09:58:29.528 INFO:tasks.rados.rados.0.smithi033.stdout:2264: snap_create 2024-09-21T09:58:29.534 INFO:tasks.rados.rados.0.smithi033.stdout:2261: finishing rollback tid 0 to smithi03385152-11 2024-09-21T09:58:29.534 INFO:tasks.rados.rados.0.smithi033.stdout:2261: finishing rollback tid 1 to smithi03385152-11 2024-09-21T09:58:29.534 INFO:tasks.rados.rados.0.smithi033.stdout:2261: finishing rollback tid 2 to smithi03385152-11 2024-09-21T09:58:29.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1032 (ObjNum 663 snap 184 seq_num 663) dirty exists 2024-09-21T09:58:29.534 INFO:tasks.rados.rados.0.smithi033.stdout:2263: finishing write tid 1 to smithi03385152-37 2024-09-21T09:58:29.535 INFO:tasks.rados.rados.0.smithi033.stdout:2263: finishing write tid 2 to smithi03385152-37 2024-09-21T09:58:29.537 INFO:tasks.rados.rados.0.smithi033.stdout:2262: finishing write tid 1 to smithi03385152-6 2024-09-21T09:58:29.537 INFO:tasks.rados.rados.0.smithi033.stdout:2263: finishing write tid 3 to smithi03385152-37 2024-09-21T09:58:29.537 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1034 (ObjNum 731 snap 210 seq_num 731) dirty exists 2024-09-21T09:58:29.537 INFO:tasks.rados.rados.0.smithi033.stdout:2263: left oid 37 (ObjNum 731 snap 210 seq_num 731) 2024-09-21T09:58:29.537 INFO:tasks.rados.rados.0.smithi033.stdout:2262: finishing write tid 2 to smithi03385152-6 2024-09-21T09:58:29.538 INFO:tasks.rados.rados.0.smithi033.stdout:2262: finishing write tid 3 to smithi03385152-6 2024-09-21T09:58:29.538 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1074 (ObjNum 730 snap 210 seq_num 730) dirty exists 2024-09-21T09:58:29.538 INFO:tasks.rados.rados.0.smithi033.stdout:2262: left oid 6 (ObjNum 730 snap 210 seq_num 730) 2024-09-21T09:58:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:29 smithi105 ceph-mon[86649]: pgmap v350: 121 pgs: 121 active+clean; 266 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 88 KiB/s wr, 3 op/s 2024-09-21T09:58:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:29 smithi105 ceph-mon[86649]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:58:29.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:29 smithi105 ceph-mon[86649]: osdmap e481: 8 total, 7 up, 8 in 2024-09-21T09:58:29.780 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:29 smithi105 podman[96419]: 2024-09-21 09:58:29.540612775 +0000 UTC m=+1.521267146 container cleanup c7fc724fff44aa8a5d9b89de39a4439de8bd4bbb60524c6c5def2afb14c7b975 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:58:29.780 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:29 smithi105 bash[96419]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5 2024-09-21T09:58:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:29 smithi033 ceph-mon[98546]: pgmap v350: 121 pgs: 121 active+clean; 266 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 88 KiB/s wr, 3 op/s 2024-09-21T09:58:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:29 smithi033 ceph-mon[98546]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:58:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:29 smithi033 ceph-mon[98546]: osdmap e481: 8 total, 7 up, 8 in 2024-09-21T09:58:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:29 smithi033 ceph-mon[96568]: pgmap v350: 121 pgs: 121 active+clean; 266 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 88 KiB/s wr, 3 op/s 2024-09-21T09:58:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:29 smithi033 ceph-mon[96568]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:58:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:29 smithi033 ceph-mon[96568]: osdmap e481: 8 total, 7 up, 8 in 2024-09-21T09:58:30.210 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:29 smithi105 podman[96441]: 2024-09-21 09:58:29.859108588 +0000 UTC m=+0.779021938 container remove c7fc724fff44aa8a5d9b89de39a4439de8bd4bbb60524c6c5def2afb14c7b975 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-21T09:58:30.473 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:30.473 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:30 smithi105 podman[96535]: 2024-09-21 09:58:30.110724203 +0000 UTC m=+0.015657859 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:58:30.473 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:30 smithi105 podman[96535]: 2024-09-21 09:58:30.209779485 +0000 UTC m=+0.114713145 container create 03d07ce296cbe484b1332caeba69a9596ea5380445c28397cd70548969e2c894 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T09:58:30.473 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:30 smithi105 podman[96535]: 2024-09-21 09:58:30.47103072 +0000 UTC m=+0.375964383 container init 03d07ce296cbe484b1332caeba69a9596ea5380445c28397cd70548969e2c894 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T09:58:30.528 INFO:tasks.rados.rados.0.smithi033.stdout:2261: done (3 left) 2024-09-21T09:58:30.528 INFO:tasks.rados.rados.0.smithi033.stdout:2262: done (2 left) 2024-09-21T09:58:30.528 INFO:tasks.rados.rados.0.smithi033.stdout:2263: done (1 left) 2024-09-21T09:58:30.528 INFO:tasks.rados.rados.0.smithi033.stdout:2264: done (0 left) 2024-09-21T09:58:30.528 INFO:tasks.rados.rados.0.smithi033.stdout:2265: rollback oid 16 current snap is 211 2024-09-21T09:58:30.528 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 16 to 198 2024-09-21T09:58:30.529 INFO:tasks.rados.rados.0.smithi033.stdout:2266: read oid 42 snap -1 2024-09-21T09:58:30.529 INFO:tasks.rados.rados.0.smithi033.stdout:2266: expect (ObjNum 683 snap 187 seq_num 683) 2024-09-21T09:58:30.529 INFO:tasks.rados.rados.0.smithi033.stdout:2267: snap_remove snap 178 2024-09-21T09:58:30.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:30 smithi105 ceph-mon[86649]: osdmap e482: 8 total, 7 up, 8 in 2024-09-21T09:58:30.781 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:30 smithi105 podman[96535]: 2024-09-21 09:58:30.475953896 +0000 UTC m=+0.380887561 container start 03d07ce296cbe484b1332caeba69a9596ea5380445c28397cd70548969e2c894 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-21T09:58:30.782 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:30 smithi105 podman[96535]: 2024-09-21 09:58:30.554067094 +0000 UTC m=+0.459000765 container attach 03d07ce296cbe484b1332caeba69a9596ea5380445c28397cd70548969e2c894 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T09:58:30.782 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:30 smithi105 podman[96535]: 2024-09-21 09:58:30.667428559 +0000 UTC m=+0.572362229 container died 03d07ce296cbe484b1332caeba69a9596ea5380445c28397cd70548969e2c894 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_BRANCH=HEAD) 2024-09-21T09:58:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:30 smithi033 ceph-mon[98546]: osdmap e482: 8 total, 7 up, 8 in 2024-09-21T09:58:30.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:30 smithi033 ceph-mon[96568]: osdmap e482: 8 total, 7 up, 8 in 2024-09-21T09:58:31.555 INFO:tasks.rados.rados.0.smithi033.stdout:2267: done (2 left) 2024-09-21T09:58:31.555 INFO:tasks.rados.rados.0.smithi033.stdout:2268: copy_from oid 12 from oid 25 current snap is 211 2024-09-21T09:58:31.555 INFO:tasks.rados.rados.0.smithi033.stdout:2269: snap_create 2024-09-21T09:58:31.555 INFO:tasks.rados.rados.0.smithi033.stdout:2265: finishing rollback tid 1 to smithi03385152-16 2024-09-21T09:58:31.555 INFO:tasks.rados.rados.0.smithi033.stdout:2265: finishing rollback tid 2 to smithi03385152-16 2024-09-21T09:58:31.555 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 1516 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:58:31.555 INFO:tasks.rados.rados.0.smithi033.stdout:2266: expect (ObjNum 683 snap 187 seq_num 683) 2024-09-21T09:58:31.659 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:31 smithi105 ceph-mon[86649]: pgmap v353: 121 pgs: 4 active+undersized, 6 peering, 15 stale+active+clean, 3 active+undersized+degraded, 93 active+clean; 265 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 104 KiB/s wr, 3 op/s; 31/510 objects degraded (6.078%) 2024-09-21T09:58:31.659 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:31 smithi105 ceph-mon[86649]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:58:31.659 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:31 smithi105 ceph-mon[86649]: Health check failed: Degraded data redundancy: 31/510 objects degraded (6.078%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:31.659 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:31 smithi105 ceph-mon[86649]: osdmap e483: 8 total, 7 up, 8 in 2024-09-21T09:58:31.659 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:31.659 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:31.659 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:31 smithi105 podman[96578]: 2024-09-21 09:58:31.382916317 +0000 UTC m=+0.709863406 container remove 03d07ce296cbe484b1332caeba69a9596ea5380445c28397cd70548969e2c894 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True) 2024-09-21T09:58:31.659 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:31 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.5.service: Deactivated successfully. 2024-09-21T09:58:31.659 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:31 smithi105 systemd[1]: Stopped Ceph osd.5 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:58:31.660 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:31 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.5.service: Consumed 19.124s CPU time. 2024-09-21T09:58:31.773 INFO:tasks.rados.rados.0.smithi033.stdout:2268: finishing copy_from racing read to smithi03385152-12 2024-09-21T09:58:31.773 INFO:tasks.rados.rados.0.smithi033.stdout:2268: finishing copy_from to smithi03385152-12 2024-09-21T09:58:31.773 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 915 (ObjNum 535 snap 143 seq_num 535) dirty exists 2024-09-21T09:58:31.917 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:31 smithi105 systemd[1]: Starting Ceph osd.5 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:58:31.917 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:31 smithi105 podman[96684]: 2024-09-21 09:58:31.78793656 +0000 UTC m=+0.016109038 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:58:31.917 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:31 smithi105 podman[96684]: 2024-09-21 09:58:31.900693394 +0000 UTC m=+0.128865871 container create ac94bcadadc3e3ca64002411cd75aa6d0e0a4d511c84068972c6a1fd20aac0ba (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:58:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:58:31] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-09-21T09:58:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[96568]: pgmap v353: 121 pgs: 4 active+undersized, 6 peering, 15 stale+active+clean, 3 active+undersized+degraded, 93 active+clean; 265 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 104 KiB/s wr, 3 op/s; 31/510 objects degraded (6.078%) 2024-09-21T09:58:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[96568]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:58:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[96568]: Health check failed: Degraded data redundancy: 31/510 objects degraded (6.078%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[96568]: osdmap e483: 8 total, 7 up, 8 in 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[98546]: pgmap v353: 121 pgs: 4 active+undersized, 6 peering, 15 stale+active+clean, 3 active+undersized+degraded, 93 active+clean; 265 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 104 KiB/s wr, 3 op/s; 31/510 objects degraded (6.078%) 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[98546]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[98546]: Health check failed: Degraded data redundancy: 31/510 objects degraded (6.078%), 3 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[98546]: osdmap e483: 8 total, 7 up, 8 in 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:32.239 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 podman[96684]: 2024-09-21 09:58:32.145059242 +0000 UTC m=+0.373231720 container init ac94bcadadc3e3ca64002411cd75aa6d0e0a4d511c84068972c6a1fd20aac0ba (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-09-21T09:58:32.239 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 podman[96684]: 2024-09-21 09:58:32.149977575 +0000 UTC m=+0.378150052 container start ac94bcadadc3e3ca64002411cd75aa6d0e0a4d511c84068972c6a1fd20aac0ba (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS) 2024-09-21T09:58:32.239 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 podman[96684]: 2024-09-21 09:58:32.236557386 +0000 UTC m=+0.464729900 container attach ac94bcadadc3e3ca64002411cd75aa6d0e0a4d511c84068972c6a1fd20aac0ba (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T09:58:32.530 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:58:32.530 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 bash[96684]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:58:32.531 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:58:32.531 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 bash[96684]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:58:32.541 INFO:tasks.rados.rados.0.smithi033.stdout:2265: done (3 left) 2024-09-21T09:58:32.541 INFO:tasks.rados.rados.0.smithi033.stdout:2266: done (2 left) 2024-09-21T09:58:32.541 INFO:tasks.rados.rados.0.smithi033.stdout:2268: done (1 left) 2024-09-21T09:58:32.541 INFO:tasks.rados.rados.0.smithi033.stdout:2269: done (0 left) 2024-09-21T09:58:32.541 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 212 2024-09-21T09:58:32.542 INFO:tasks.rados.rados.0.smithi033.stdout:2270: seq_num 732 ranges {0=262144} 2024-09-21T09:58:32.542 INFO:tasks.rados.rados.0.smithi033.stdout:2270: writing smithi03385152-40 from 0 to 262144 tid 1 2024-09-21T09:58:32.542 INFO:tasks.rados.rados.0.smithi033.stdout:2271: copy_from oid 27 from oid 9 current snap is 212 2024-09-21T09:58:32.543 INFO:tasks.rados.rados.0.smithi033.stdout:2272: snap_remove snap 198 2024-09-21T09:58:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:32 smithi033 ceph-mon[98546]: osdmap e484: 8 total, 7 up, 8 in 2024-09-21T09:58:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:32 smithi033 ceph-mon[96568]: osdmap e484: 8 total, 7 up, 8 in 2024-09-21T09:58:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:32 smithi105 ceph-mon[86649]: osdmap e484: 8 total, 7 up, 8 in 2024-09-21T09:58:33.030 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T09:58:33.030 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 bash[96684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T09:58:33.030 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:58:33.030 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:32 smithi105 bash[96684]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-21T09:58:33.442 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:58:33.442 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 bash[96684]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-21T09:58:33.442 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:58:33.442 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 bash[96684]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-21T09:58:33.443 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T09:58:33.443 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 bash[96684]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-09-21T09:58:33.443 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T09:58:33.443 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 bash[96684]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-09-21T09:58:33.443 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate[96707]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T09:58:33.443 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 bash[96684]: --> ceph-volume raw activate successful for osd ID: 5 2024-09-21T09:58:33.443 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 podman[96684]: 2024-09-21 09:58:33.066099235 +0000 UTC m=+1.294271713 container died ac94bcadadc3e3ca64002411cd75aa6d0e0a4d511c84068972c6a1fd20aac0ba (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True) 2024-09-21T09:58:33.497 INFO:tasks.rados.rados.0.smithi033.stdout:2272: done (2 left) 2024-09-21T09:58:33.497 INFO:tasks.rados.rados.0.smithi033.stdout:2273: copy_from oid 32 from oid 2 current snap is 212 2024-09-21T09:58:33.497 INFO:tasks.rados.rados.0.smithi033.stdout:2271: finishing copy_from to smithi03385152-27 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2271: got expected ENOENT (src dne) 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2271: finishing copy_from racing read to smithi03385152-27 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2270: finishing write tid 1 to smithi03385152-40 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2270: finishing write tid 2 to smithi03385152-40 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2270: finishing write tid 3 to smithi03385152-40 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 789 (ObjNum 732 snap 212 seq_num 732) dirty exists 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2270: left oid 40 (ObjNum 732 snap 212 seq_num 732) 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2270: done (2 left) 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2271: done (1 left) 2024-09-21T09:58:33.498 INFO:tasks.rados.rados.0.smithi033.stdout:2274: rmattr oid 44 current snap is 212 2024-09-21T09:58:33.499 INFO:tasks.rados.rados.0.smithi033.stdout:2275: copy_from oid 21 from oid 29 current snap is 212 2024-09-21T09:58:33.499 INFO:tasks.rados.rados.0.smithi033.stdout:2276: snap_create 2024-09-21T09:58:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2273: finishing copy_from to smithi03385152-32 2024-09-21T09:58:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2273: got expected ENOENT (src dne) 2024-09-21T09:58:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2273: finishing copy_from racing read to smithi03385152-32 2024-09-21T09:58:33.509 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1040 (ObjNum 728 snap 210 seq_num 728) dirty exists 2024-09-21T09:58:33.567 INFO:tasks.rados.rados.0.smithi033.stdout:2275: finishing copy_from racing read to smithi03385152-21 2024-09-21T09:58:33.571 INFO:tasks.rados.rados.0.smithi033.stdout:2275: finishing copy_from to smithi03385152-21 2024-09-21T09:58:33.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1469 (ObjNum 713 snap 200 seq_num 713) dirty exists 2024-09-21T09:58:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:33 smithi105 ceph-mon[86649]: pgmap v356: 121 pgs: 23 active+undersized, 6 peering, 6 stale+active+clean, 11 active+undersized+degraded, 75 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 208 KiB/s wr, 13 op/s; 76/495 objects degraded (15.354%) 2024-09-21T09:58:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:33 smithi105 ceph-mon[86649]: osdmap e485: 8 total, 7 up, 8 in 2024-09-21T09:58:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:33 smithi105 ceph-mon[86649]: osdmap e486: 8 total, 7 up, 8 in 2024-09-21T09:58:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:33 smithi033 ceph-mon[96568]: pgmap v356: 121 pgs: 23 active+undersized, 6 peering, 6 stale+active+clean, 11 active+undersized+degraded, 75 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 208 KiB/s wr, 13 op/s; 76/495 objects degraded (15.354%) 2024-09-21T09:58:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:33 smithi033 ceph-mon[96568]: osdmap e485: 8 total, 7 up, 8 in 2024-09-21T09:58:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:33 smithi033 ceph-mon[96568]: osdmap e486: 8 total, 7 up, 8 in 2024-09-21T09:58:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:33 smithi033 ceph-mon[98546]: pgmap v356: 121 pgs: 23 active+undersized, 6 peering, 6 stale+active+clean, 11 active+undersized+degraded, 75 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 208 KiB/s wr, 13 op/s; 76/495 objects degraded (15.354%) 2024-09-21T09:58:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:33 smithi033 ceph-mon[98546]: osdmap e485: 8 total, 7 up, 8 in 2024-09-21T09:58:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:33 smithi033 ceph-mon[98546]: osdmap e486: 8 total, 7 up, 8 in 2024-09-21T09:58:34.281 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:33 smithi105 podman[96855]: 2024-09-21 09:58:33.991602176 +0000 UTC m=+0.919973980 container remove ac94bcadadc3e3ca64002411cd75aa6d0e0a4d511c84068972c6a1fd20aac0ba (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-09-21T09:58:34.282 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:34 smithi105 podman[96923]: 2024-09-21 09:58:34.200714043 +0000 UTC m=+0.110571846 container create 4ff1257a8be148977636a39842a9a0a3a38e2b333dbe1223f8d1fc4403c623c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-62da54b, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:58:34.282 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:34 smithi105 podman[96923]: 2024-09-21 09:58:34.105987357 +0000 UTC m=+0.015845168 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:58:34.494 INFO:tasks.rados.rados.0.smithi033.stdout:2273: done (3 left) 2024-09-21T09:58:34.495 INFO:tasks.rados.rados.0.smithi033.stdout:2274: done (2 left) 2024-09-21T09:58:34.495 INFO:tasks.rados.rados.0.smithi033.stdout:2275: done (1 left) 2024-09-21T09:58:34.495 INFO:tasks.rados.rados.0.smithi033.stdout:2276: done (0 left) 2024-09-21T09:58:34.495 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 213 2024-09-21T09:58:34.495 INFO:tasks.rados.rados.0.smithi033.stdout:2277: seq_num 733 ranges {262144=278528} 2024-09-21T09:58:34.497 INFO:tasks.rados.rados.0.smithi033.stdout:2277: writing smithi03385152-40 from 262144 to 540672 tid 1 2024-09-21T09:58:34.497 INFO:tasks.rados.rados.0.smithi033.stdout:2278: setattr oid 23 current snap is 213 2024-09-21T09:58:34.498 INFO:tasks.rados.rados.0.smithi033.stdout:2279: delete oid 21 current snap is 213 2024-09-21T09:58:34.505 INFO:tasks.rados.rados.0.smithi033.stdout:2277: finishing write tid 1 to smithi03385152-40 2024-09-21T09:58:34.505 INFO:tasks.rados.rados.0.smithi033.stdout:2277: finishing write tid 2 to smithi03385152-40 2024-09-21T09:58:34.507 INFO:tasks.rados.rados.0.smithi033.stdout:2277: finishing write tid 3 to smithi03385152-40 2024-09-21T09:58:34.507 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 796 (ObjNum 733 snap 213 seq_num 733) dirty exists 2024-09-21T09:58:34.507 INFO:tasks.rados.rados.0.smithi033.stdout:2277: left oid 40 (ObjNum 733 snap 213 seq_num 733) 2024-09-21T09:58:34.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1470 (ObjNum 689 snap 192 seq_num 689) dirty exists 2024-09-21T09:58:34.509 INFO:tasks.rados.rados.0.smithi033.stdout:2277: done (2 left) 2024-09-21T09:58:34.509 INFO:tasks.rados.rados.0.smithi033.stdout:2278: done (1 left) 2024-09-21T09:58:34.509 INFO:tasks.rados.rados.0.smithi033.stdout:2279: done (0 left) 2024-09-21T09:58:34.509 INFO:tasks.rados.rados.0.smithi033.stdout:2280: read oid 41 snap -1 2024-09-21T09:58:34.509 INFO:tasks.rados.rados.0.smithi033.stdout:2280: expect (ObjNum 692 snap 192 seq_num 692) 2024-09-21T09:58:34.509 INFO:tasks.rados.rados.0.smithi033.stdout:2281: rollback oid 4 current snap is 213 2024-09-21T09:58:34.509 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 4 to 202 2024-09-21T09:58:34.510 INFO:tasks.rados.rados.0.smithi033.stdout:2282: setattr oid 3 current snap is 213 2024-09-21T09:58:34.512 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 213 2024-09-21T09:58:34.512 INFO:tasks.rados.rados.0.smithi033.stdout:2283: seq_num 735 ranges {3489792=311296} 2024-09-21T09:58:34.534 INFO:tasks.rados.rados.0.smithi033.stdout:2283: writing smithi03385152-10 from 3489792 to 3801088 tid 1 2024-09-21T09:58:34.534 INFO:tasks.rados.rados.0.smithi033.stdout:2284: copy_from oid 47 from oid 11 current snap is 213 2024-09-21T09:58:34.534 INFO:tasks.rados.rados.0.smithi033.stdout:2281: finishing rollback tid 0 to smithi03385152-4 2024-09-21T09:58:34.534 INFO:tasks.rados.rados.0.smithi033.stdout:2285: snap_remove snap 194 2024-09-21T09:58:34.534 INFO:tasks.rados.rados.0.smithi033.stdout:2281: finishing rollback tid 1 to smithi03385152-4 2024-09-21T09:58:34.534 INFO:tasks.rados.rados.0.smithi033.stdout:2281: finishing rollback tid 2 to smithi03385152-4 2024-09-21T09:58:34.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1043 (ObjNum 712 snap 199 seq_num 712) dirty exists 2024-09-21T09:58:34.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1194 (ObjNum 450 snap 120 seq_num 450) dirty exists 2024-09-21T09:58:34.780 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:34 smithi105 podman[96923]: 2024-09-21 09:58:34.43677781 +0000 UTC m=+0.346635615 container init 4ff1257a8be148977636a39842a9a0a3a38e2b333dbe1223f8d1fc4403c623c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-21T09:58:34.781 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:34 smithi105 podman[96923]: 2024-09-21 09:58:34.441520792 +0000 UTC m=+0.351378604 container start 4ff1257a8be148977636a39842a9a0a3a38e2b333dbe1223f8d1fc4403c623c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T09:58:34.781 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:34 smithi105 ceph-osd[96961]: -- 172.21.15.105:0/3659119765 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55647e2dfa40 con 0x55647f0ae400 2024-09-21T09:58:34.781 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:34 smithi105 bash[96923]: 4ff1257a8be148977636a39842a9a0a3a38e2b333dbe1223f8d1fc4403c623c4 2024-09-21T09:58:34.781 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:34 smithi105 systemd[1]: Started Ceph osd.5 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:58:35.499 INFO:tasks.rados.rados.0.smithi033.stdout:2281: done (5 left) 2024-09-21T09:58:35.499 INFO:tasks.rados.rados.0.smithi033.stdout:2282: done (4 left) 2024-09-21T09:58:35.499 INFO:tasks.rados.rados.0.smithi033.stdout:2285: done (3 left) 2024-09-21T09:58:35.499 INFO:tasks.rados.rados.0.smithi033.stdout:2286: snap_create 2024-09-21T09:58:35.499 INFO:tasks.rados.rados.0.smithi033.stdout:2280: expect (ObjNum 692 snap 192 seq_num 692) 2024-09-21T09:58:35.613 INFO:tasks.rados.rados.0.smithi033.stdout:2283: finishing write tid 1 to smithi03385152-10 2024-09-21T09:58:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:2283: finishing write tid 2 to smithi03385152-10 2024-09-21T09:58:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:2283: finishing write tid 3 to smithi03385152-10 2024-09-21T09:58:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1024 (ObjNum 735 snap 213 seq_num 735) dirty exists 2024-09-21T09:58:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:2283: left oid 10 (ObjNum 735 snap 213 seq_num 735) 2024-09-21T09:58:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:2284: finishing copy_from racing read to smithi03385152-47 2024-09-21T09:58:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:2284: finishing copy_from to smithi03385152-47 2024-09-21T09:58:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1032 (ObjNum 663 snap 184 seq_num 663) dirty exists 2024-09-21T09:58:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[98546]: pgmap v359: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 168 KiB/s wr, 13 op/s; 85/489 objects degraded (17.382%) 2024-09-21T09:58:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[98546]: osdmap e487: 8 total, 7 up, 8 in 2024-09-21T09:58:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[98546]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:58:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:35.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[96568]: pgmap v359: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 168 KiB/s wr, 13 op/s; 85/489 objects degraded (17.382%) 2024-09-21T09:58:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[96568]: osdmap e487: 8 total, 7 up, 8 in 2024-09-21T09:58:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[96568]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:58:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:35 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:35 smithi105 ceph-mon[86649]: pgmap v359: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 252 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 168 KiB/s wr, 13 op/s; 85/489 objects degraded (17.382%) 2024-09-21T09:58:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:35 smithi105 ceph-mon[86649]: osdmap e487: 8 total, 7 up, 8 in 2024-09-21T09:58:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:35 smithi105 ceph-mon[86649]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:58:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:35 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:36.502 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T09:58:36.077+0000 7f42d96f9640 -1 Falling back to public interface 2024-09-21T09:58:36.508 INFO:tasks.rados.rados.0.smithi033.stdout:2280: done (3 left) 2024-09-21T09:58:36.508 INFO:tasks.rados.rados.0.smithi033.stdout:2283: done (2 left) 2024-09-21T09:58:36.508 INFO:tasks.rados.rados.0.smithi033.stdout:2284: done (1 left) 2024-09-21T09:58:36.509 INFO:tasks.rados.rados.0.smithi033.stdout:2286: done (0 left) 2024-09-21T09:58:36.509 INFO:tasks.rados.rados.0.smithi033.stdout:2287: rollback oid 29 current snap is 214 2024-09-21T09:58:36.509 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 29 to 192 2024-09-21T09:58:36.509 INFO:tasks.rados.rados.0.smithi033.stdout:2288: snap_remove snap 213 2024-09-21T09:58:36.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:36 smithi105 ceph-mon[86649]: osdmap e488: 8 total, 7 up, 8 in 2024-09-21T09:58:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:36 smithi033 ceph-mon[98546]: osdmap e488: 8 total, 7 up, 8 in 2024-09-21T09:58:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:36 smithi033 ceph-mon[96568]: osdmap e488: 8 total, 7 up, 8 in 2024-09-21T09:58:37.511 INFO:tasks.rados.rados.0.smithi033.stdout:2288: done (1 left) 2024-09-21T09:58:37.512 INFO:tasks.rados.rados.0.smithi033.stdout:2289: rollback oid 42 current snap is 214 2024-09-21T09:58:37.512 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 42 to 197 2024-09-21T09:58:37.512 INFO:tasks.rados.rados.0.smithi033.stdout:2290: delete oid 2 current snap is 214 2024-09-21T09:58:37.512 INFO:tasks.rados.rados.0.smithi033.stdout:2287: finishing rollback tid 0 to smithi03385152-29 2024-09-21T09:58:37.512 INFO:tasks.rados.rados.0.smithi033.stdout:2287: finishing rollback tid 1 to smithi03385152-29 2024-09-21T09:58:37.512 INFO:tasks.rados.rados.0.smithi033.stdout:2287: finishing rollback tid 2 to smithi03385152-29 2024-09-21T09:58:37.512 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1520 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:58:37.520 INFO:tasks.rados.rados.0.smithi033.stdout:2289: finishing rollback tid 0 to smithi03385152-42 2024-09-21T09:58:37.520 INFO:tasks.rados.rados.0.smithi033.stdout:2289: finishing rollback tid 1 to smithi03385152-42 2024-09-21T09:58:37.520 INFO:tasks.rados.rados.0.smithi033.stdout:2289: finishing rollback tid 2 to smithi03385152-42 2024-09-21T09:58:37.520 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 802 (ObjNum 683 snap 187 seq_num 683) dirty exists 2024-09-21T09:58:37.521 INFO:tasks.rados.rados.0.smithi033.stdout:2287: done (2 left) 2024-09-21T09:58:37.521 INFO:tasks.rados.rados.0.smithi033.stdout:2289: done (1 left) 2024-09-21T09:58:37.521 INFO:tasks.rados.rados.0.smithi033.stdout:2290: done (0 left) 2024-09-21T09:58:37.521 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 214 2024-09-21T09:58:37.521 INFO:tasks.rados.rados.0.smithi033.stdout:2291: seq_num 737 ranges {1040384=352256} 2024-09-21T09:58:37.529 INFO:tasks.rados.rados.0.smithi033.stdout:2291: writing smithi03385152-34 from 1040384 to 1392640 tid 1 2024-09-21T09:58:37.529 INFO:tasks.rados.rados.0.smithi033.stdout:2292: rmattr oid 44 current snap is 214 2024-09-21T09:58:37.529 INFO:tasks.rados.rados.0.smithi033.stdout:2293: rmattr oid 10 current snap is 214 2024-09-21T09:58:37.529 INFO:tasks.rados.rados.0.smithi033.stdout:2293: done (2 left) 2024-09-21T09:58:37.530 INFO:tasks.rados.rados.0.smithi033.stdout:2294: copy_from oid 23 from oid 37 current snap is 214 2024-09-21T09:58:37.530 INFO:tasks.rados.rados.0.smithi033.stdout:2295: copy_from oid 11 from oid 50 current snap is 214 2024-09-21T09:58:37.530 INFO:tasks.rados.rados.0.smithi033.stdout:2296: snap_remove snap 206 2024-09-21T09:58:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:37 smithi105 ceph-mon[86649]: pgmap v362: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 270 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 4.5 MiB/s wr, 14 op/s; 87/498 objects degraded (17.470%) 2024-09-21T09:58:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:37 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 87/498 objects degraded (17.470%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:37 smithi105 ceph-mon[86649]: osdmap e489: 8 total, 7 up, 8 in 2024-09-21T09:58:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:37 smithi033 ceph-mon[98546]: pgmap v362: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 270 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 4.5 MiB/s wr, 14 op/s; 87/498 objects degraded (17.470%) 2024-09-21T09:58:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:37 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 87/498 objects degraded (17.470%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:37 smithi033 ceph-mon[98546]: osdmap e489: 8 total, 7 up, 8 in 2024-09-21T09:58:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:37 smithi033 ceph-mon[96568]: pgmap v362: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 270 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 4.5 MiB/s wr, 14 op/s; 87/498 objects degraded (17.470%) 2024-09-21T09:58:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:37 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 87/498 objects degraded (17.470%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:37 smithi033 ceph-mon[96568]: osdmap e489: 8 total, 7 up, 8 in 2024-09-21T09:58:38.518 INFO:tasks.rados.rados.0.smithi033.stdout:2296: done (4 left) 2024-09-21T09:58:38.518 INFO:tasks.rados.rados.0.smithi033.stdout:2297: rollback oid 1 current snap is 214 2024-09-21T09:58:38.518 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 204 2024-09-21T09:58:38.518 INFO:tasks.rados.rados.0.smithi033.stdout:2295: finishing copy_from to smithi03385152-11 2024-09-21T09:58:38.518 INFO:tasks.rados.rados.0.smithi033.stdout:2295: got expected ENOENT (src dne) 2024-09-21T09:58:38.518 INFO:tasks.rados.rados.0.smithi033.stdout:2295: finishing copy_from racing read to smithi03385152-11 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1044 (ObjNum 728 snap 210 seq_num 728) dirty exists 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2291: finishing write tid 1 to smithi03385152-34 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2291: finishing write tid 2 to smithi03385152-34 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2294: finishing copy_from racing read to smithi03385152-23 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2294: finishing copy_from to smithi03385152-23 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1034 (ObjNum 731 snap 210 seq_num 731) dirty exists 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2291: finishing write tid 3 to smithi03385152-34 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1197 (ObjNum 737 snap 214 seq_num 737) dirty exists 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2291: left oid 34 (ObjNum 737 snap 214 seq_num 737) 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2291: done (4 left) 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2292: done (3 left) 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2294: done (2 left) 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2295: done (1 left) 2024-09-21T09:58:38.519 INFO:tasks.rados.rados.0.smithi033.stdout:2298: rollback oid 9 current snap is 214 2024-09-21T09:58:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 9 to 201 2024-09-21T09:58:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:2299: read oid 48 snap -1 2024-09-21T09:58:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:2299: expect (ObjNum 691 snap 192 seq_num 691) 2024-09-21T09:58:38.520 INFO:tasks.rados.rados.0.smithi033.stdout:2300: setattr oid 27 current snap is 214 2024-09-21T09:58:38.521 INFO:tasks.rados.rados.0.smithi033.stdout:2301: rollback oid 17 current snap is 214 2024-09-21T09:58:38.521 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 197 2024-09-21T09:58:38.521 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 214 2024-09-21T09:58:38.521 INFO:tasks.rados.rados.0.smithi033.stdout:2302: seq_num 738 ranges {4308992=368640} 2024-09-21T09:58:38.547 INFO:tasks.rados.rados.0.smithi033.stdout:2302: writing smithi03385152-36 from 4308992 to 4677632 tid 1 2024-09-21T09:58:38.548 INFO:tasks.rados.rados.0.smithi033.stdout:2303: read oid 35 snap 208 2024-09-21T09:58:38.548 INFO:tasks.rados.rados.0.smithi033.stdout:2303: expect (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:58:38.548 INFO:tasks.rados.rados.0.smithi033.stdout:2304: rollback oid 46 current snap is 214 2024-09-21T09:58:38.548 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 46 to 212 2024-09-21T09:58:38.548 INFO:tasks.rados.rados.0.smithi033.stdout:2301: finishing rollback tid 1 to smithi03385152-17 2024-09-21T09:58:38.548 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 1099 (ObjNum 129 snap 0 seq_num 91664288) dirty dne 2024-09-21T09:58:38.548 INFO:tasks.rados.rados.0.smithi033.stdout:2301: done (7 left) 2024-09-21T09:58:38.548 INFO:tasks.rados.rados.0.smithi033.stdout:2305: snap_remove snap 192 2024-09-21T09:58:38.777 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:38 smithi105 ceph-mon[86649]: osdmap e490: 8 total, 7 up, 8 in 2024-09-21T09:58:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:38 smithi033 ceph-mon[98546]: osdmap e490: 8 total, 7 up, 8 in 2024-09-21T09:58:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:38 smithi033 ceph-mon[96568]: osdmap e490: 8 total, 7 up, 8 in 2024-09-21T09:58:39.525 INFO:tasks.rados.rados.0.smithi033.stdout:2305: done (7 left) 2024-09-21T09:58:39.525 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 214 2024-09-21T09:58:39.525 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1075 (ObjNum 697 snap 193 seq_num 697) dirty exists 2024-09-21T09:58:39.525 INFO:tasks.rados.rados.0.smithi033.stdout:2306: seq_num 739 ranges {0=385024} 2024-09-21T09:58:39.527 INFO:tasks.rados.rados.0.smithi033.stdout:2306: writing smithi03385152-21 from 0 to 385024 tid 1 2024-09-21T09:58:39.527 INFO:tasks.rados.rados.0.smithi033.stdout:2300: done (7 left) 2024-09-21T09:58:39.527 INFO:tasks.rados.rados.0.smithi033.stdout:2307: rollback oid 40 current snap is 214 2024-09-21T09:58:39.527 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 40 to 202 2024-09-21T09:58:39.527 INFO:tasks.rados.rados.0.smithi033.stdout:2297: finishing rollback tid 0 to smithi03385152-1 2024-09-21T09:58:39.528 INFO:tasks.rados.rados.0.smithi033.stdout:2297: finishing rollback tid 1 to smithi03385152-1 2024-09-21T09:58:39.528 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 921 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:58:39.528 INFO:tasks.rados.rados.0.smithi033.stdout:2299: expect (ObjNum 691 snap 192 seq_num 691) 2024-09-21T09:58:39.593 INFO:tasks.rados.rados.0.smithi033.stdout:2298: finishing rollback tid 1 to smithi03385152-9 2024-09-21T09:58:39.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 778 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T09:58:39.593 INFO:tasks.rados.rados.0.smithi033.stdout:2304: finishing rollback tid 0 to smithi03385152-46 2024-09-21T09:58:39.593 INFO:tasks.rados.rados.0.smithi033.stdout:2304: finishing rollback tid 1 to smithi03385152-46 2024-09-21T09:58:39.593 INFO:tasks.rados.rados.0.smithi033.stdout:2297: done (7 left) 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2298: done (6 left) 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2299: done (5 left) 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2308: delete oid 23 current snap is 214 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2304: finishing rollback tid 2 to smithi03385152-46 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1201 (ObjNum 667 snap 185 seq_num 667) dirty exists 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2302: finishing write tid 1 to smithi03385152-36 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2302: finishing write tid 2 to smithi03385152-36 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2302: finishing write tid 3 to smithi03385152-36 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1523 (ObjNum 738 snap 214 seq_num 738) dirty exists 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2302: left oid 36 (ObjNum 738 snap 214 seq_num 738) 2024-09-21T09:58:39.594 INFO:tasks.rados.rados.0.smithi033.stdout:2303: expect (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:58:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:39 smithi105 ceph-mon[86649]: pgmap v365: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.4 MiB/s wr, 17 op/s; 88/495 objects degraded (17.778%) 2024-09-21T09:58:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:39 smithi105 ceph-mon[86649]: osdmap e491: 8 total, 7 up, 8 in 2024-09-21T09:58:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:39 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:39 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:39 smithi033 ceph-mon[96568]: pgmap v365: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.4 MiB/s wr, 17 op/s; 88/495 objects degraded (17.778%) 2024-09-21T09:58:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:39 smithi033 ceph-mon[96568]: osdmap e491: 8 total, 7 up, 8 in 2024-09-21T09:58:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:39 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:39 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:39 smithi033 ceph-mon[98546]: pgmap v365: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 266 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.4 MiB/s wr, 17 op/s; 88/495 objects degraded (17.778%) 2024-09-21T09:58:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:39 smithi033 ceph-mon[98546]: osdmap e491: 8 total, 7 up, 8 in 2024-09-21T09:58:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:39 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:39.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:39 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:40.214 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T09:58:40.163+0000 7f42d96f9640 -1 osd.5 480 log_to_monitors true 2024-09-21T09:58:40.520 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:40.780 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 09:58:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T09:58:40.610+0000 7f42d14a3640 -1 osd.5 480 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:58:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:40 smithi105 ceph-mon[86649]: osdmap e492: 8 total, 7 up, 8 in 2024-09-21T09:58:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:40 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:40 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:40 smithi105 ceph-mon[86649]: from='osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:58:40.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:40 smithi105 ceph-mon[86649]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2306: finishing write tid 1 to smithi03385152-21 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2306: finishing write tid 2 to smithi03385152-21 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2306: finishing write tid 3 to smithi03385152-21 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1473 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2306: left oid 21 (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2307: finishing rollback tid 0 to smithi03385152-40 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2307: finishing rollback tid 1 to smithi03385152-40 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2307: finishing rollback tid 2 to smithi03385152-40 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 812 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2302: done (5 left) 2024-09-21T09:58:40.804 INFO:tasks.rados.rados.0.smithi033.stdout:2303: done (4 left) 2024-09-21T09:58:40.805 INFO:tasks.rados.rados.0.smithi033.stdout:2304: done (3 left) 2024-09-21T09:58:40.805 INFO:tasks.rados.rados.0.smithi033.stdout:2306: done (2 left) 2024-09-21T09:58:40.805 INFO:tasks.rados.rados.0.smithi033.stdout:2307: done (1 left) 2024-09-21T09:58:40.805 INFO:tasks.rados.rados.0.smithi033.stdout:2308: done (0 left) 2024-09-21T09:58:40.805 INFO:tasks.rados.rados.0.smithi033.stdout:2309: rollback oid 25 current snap is 214 2024-09-21T09:58:40.805 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 25 to 207 2024-09-21T09:58:40.805 INFO:tasks.rados.rados.0.smithi033.stdout:2310: snap_remove snap 201 2024-09-21T09:58:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[98546]: osdmap e492: 8 total, 7 up, 8 in 2024-09-21T09:58:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[98546]: from='osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:58:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[98546]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:58:40.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[96568]: osdmap e492: 8 total, 7 up, 8 in 2024-09-21T09:58:40.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:40.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:40.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[96568]: from='osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698]' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:58:40.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:40 smithi033 ceph-mon[96568]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]} : dispatch 2024-09-21T09:58:41.587 INFO:tasks.rados.rados.0.smithi033.stdout:2310: done (1 left) 2024-09-21T09:58:41.587 INFO:tasks.rados.rados.0.smithi033.stdout:2311: read oid 35 snap -1 2024-09-21T09:58:41.587 INFO:tasks.rados.rados.0.smithi033.stdout:2311: expect (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:58:41.587 INFO:tasks.rados.rados.0.smithi033.stdout:2312: copy_from oid 13 from oid 23 current snap is 214 2024-09-21T09:58:41.587 INFO:tasks.rados.rados.0.smithi033.stdout:2313: snap_create 2024-09-21T09:58:41.587 INFO:tasks.rados.rados.0.smithi033.stdout:2309: finishing rollback tid 0 to smithi03385152-25 2024-09-21T09:58:41.587 INFO:tasks.rados.rados.0.smithi033.stdout:2309: finishing rollback tid 1 to smithi03385152-25 2024-09-21T09:58:41.588 INFO:tasks.rados.rados.0.smithi033.stdout:2309: finishing rollback tid 2 to smithi03385152-25 2024-09-21T09:58:41.588 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 925 (ObjNum 535 snap 143 seq_num 535) dirty exists 2024-09-21T09:58:41.978 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:41 smithi105 ceph-mon[86649]: pgmap v368: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 273 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.6 MiB/s wr, 10 op/s; 91/513 objects degraded (17.739%) 2024-09-21T09:58:41.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:41 smithi105 ceph-mon[86649]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:58:41.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:41 smithi105 ceph-mon[86649]: osdmap e493: 8 total, 7 up, 8 in 2024-09-21T09:58:41.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:41 smithi105 ceph-mon[86649]: from='osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:41.979 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:41 smithi105 ceph-mon[86649]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:58:41] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.51.0" 2024-09-21T09:58:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[98546]: pgmap v368: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 273 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.6 MiB/s wr, 10 op/s; 91/513 objects degraded (17.739%) 2024-09-21T09:58:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[98546]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:58:41.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[98546]: osdmap e493: 8 total, 7 up, 8 in 2024-09-21T09:58:41.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[98546]: from='osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:41.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[98546]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[96568]: pgmap v368: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 273 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 1.6 MiB/s wr, 10 op/s; 91/513 objects degraded (17.739%) 2024-09-21T09:58:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[96568]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-21T09:58:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[96568]: osdmap e493: 8 total, 7 up, 8 in 2024-09-21T09:58:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[96568]: from='osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698]' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:41 smithi033 ceph-mon[96568]: from='osd.5 ' entity='osd.5' cmd={"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:58:42.592 INFO:tasks.rados.rados.0.smithi033.stdout:2309: done (3 left) 2024-09-21T09:58:42.592 INFO:tasks.rados.rados.0.smithi033.stdout:2313: done (2 left) 2024-09-21T09:58:42.592 INFO:tasks.rados.rados.0.smithi033.stdout:2314: setattr oid 10 current snap is 215 2024-09-21T09:58:42.593 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 215 2024-09-21T09:58:42.593 INFO:tasks.rados.rados.0.smithi033.stdout:2315: seq_num 741 ranges {0=417792} 2024-09-21T09:58:42.595 INFO:tasks.rados.rados.0.smithi033.stdout:2315: writing smithi03385152-19 from 0 to 417792 tid 1 2024-09-21T09:58:42.596 INFO:tasks.rados.rados.0.smithi033.stdout:2316: copy_from oid 5 from oid 28 current snap is 215 2024-09-21T09:58:42.596 INFO:tasks.rados.rados.0.smithi033.stdout:2317: read oid 42 snap -1 2024-09-21T09:58:42.596 INFO:tasks.rados.rados.0.smithi033.stdout:2317: expect (ObjNum 683 snap 187 seq_num 683) 2024-09-21T09:58:42.597 INFO:tasks.rados.rados.0.smithi033.stdout:2318: read oid 37 snap -1 2024-09-21T09:58:42.597 INFO:tasks.rados.rados.0.smithi033.stdout:2318: expect (ObjNum 731 snap 210 seq_num 731) 2024-09-21T09:58:42.597 INFO:tasks.rados.rados.0.smithi033.stdout:2319: rollback oid 17 current snap is 215 2024-09-21T09:58:42.597 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 208 2024-09-21T09:58:42.597 INFO:tasks.rados.rados.0.smithi033.stdout:2320: rmattr oid 16 current snap is 215 2024-09-21T09:58:42.597 INFO:tasks.rados.rados.0.smithi033.stdout:2320: done (8 left) 2024-09-21T09:58:42.597 INFO:tasks.rados.rados.0.smithi033.stdout:2321: delete oid 47 current snap is 215 2024-09-21T09:58:42.603 INFO:tasks.rados.rados.0.smithi033.stdout:2319: finishing rollback tid 1 to smithi03385152-17 2024-09-21T09:58:42.603 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 1099 (ObjNum 305 snap 0 seq_num 0) dirty dne 2024-09-21T09:58:42.606 INFO:tasks.rados.rados.0.smithi033.stdout:2318: expect (ObjNum 731 snap 210 seq_num 731) 2024-09-21T09:58:42.617 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1025 (ObjNum 735 snap 213 seq_num 735) dirty exists 2024-09-21T09:58:42.620 INFO:tasks.rados.rados.0.smithi033.stdout:2315: finishing write tid 1 to smithi03385152-19 2024-09-21T09:58:42.620 INFO:tasks.rados.rados.0.smithi033.stdout:2315: finishing write tid 2 to smithi03385152-19 2024-09-21T09:58:42.620 INFO:tasks.rados.rados.0.smithi033.stdout:2315: finishing write tid 3 to smithi03385152-19 2024-09-21T09:58:42.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1027 (ObjNum 741 snap 215 seq_num 741) dirty exists 2024-09-21T09:58:42.620 INFO:tasks.rados.rados.0.smithi033.stdout:2315: left oid 19 (ObjNum 741 snap 215 seq_num 741) 2024-09-21T09:58:42.628 INFO:tasks.rados.rados.0.smithi033.stdout:2312: finishing copy_from to smithi03385152-13 2024-09-21T09:58:42.628 INFO:tasks.rados.rados.0.smithi033.stdout:2312: got expected ENOENT (src dne) 2024-09-21T09:58:42.628 INFO:tasks.rados.rados.0.smithi033.stdout:2312: finishing copy_from racing read to smithi03385152-13 2024-09-21T09:58:42.632 INFO:tasks.rados.rados.0.smithi033.stdout:2317: expect (ObjNum 683 snap 187 seq_num 683) 2024-09-21T09:58:42.875 INFO:tasks.rados.rados.0.smithi033.stdout:2316: finishing copy_from racing read to smithi03385152-5 2024-09-21T09:58:42.875 INFO:tasks.rados.rados.0.smithi033.stdout:2316: finishing copy_from to smithi03385152-5 2024-09-21T09:58:42.875 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 885 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:58:42.875 INFO:tasks.rados.rados.0.smithi033.stdout:2311: expect (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:58:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-mon[98546]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:58:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-mon[98546]: osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698] boot 2024-09-21T09:58:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-mon[98546]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:58:42.898 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:58:42.898 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:58:42.799+0000 7fc38e955640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:58:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-mon[96568]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:58:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-mon[96568]: osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698] boot 2024-09-21T09:58:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-mon[96568]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:58:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:42 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:58:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:42 smithi105 ceph-mon[86649]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:58:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:42 smithi105 ceph-mon[86649]: osd.5 [v2:172.21.15.105:6808/1966612698,v1:172.21.15.105:6809/1966612698] boot 2024-09-21T09:58:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:42 smithi105 ceph-mon[86649]: osdmap e494: 8 total, 8 up, 8 in 2024-09-21T09:58:43.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:42 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 5} : dispatch 2024-09-21T09:58:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: pgmap v371: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 18 op/s; 95/528 objects degraded (17.992%) 2024-09-21T09:58:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:58:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 95/528 objects degraded (17.992%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[96568]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: pgmap v371: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 18 op/s; 95/528 objects degraded (17.992%) 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 95/528 objects degraded (17.992%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:43.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:58:43.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:43 smithi033 ceph-mon[98546]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:58:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: pgmap v371: 121 pgs: 35 active+undersized, 14 active+undersized+degraded, 72 active+clean; 274 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 1.9 MiB/s wr, 18 op/s; 95/528 objects degraded (17.992%) 2024-09-21T09:58:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: osdmap e495: 8 total, 8 up, 8 in 2024-09-21T09:58:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 95/528 objects degraded (17.992%), 14 pgs degraded (PG_DEGRADED) 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:58:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:43 smithi105 ceph-mon[86649]: Upgrade: unsafe to stop osd(s) at this time (18 PGs are or would become offline) 2024-09-21T09:58:44.094 INFO:tasks.rados.rados.0.smithi033.stdout:2311: done (8 left) 2024-09-21T09:58:44.094 INFO:tasks.rados.rados.0.smithi033.stdout:2312: done (7 left) 2024-09-21T09:58:44.094 INFO:tasks.rados.rados.0.smithi033.stdout:2314: done (6 left) 2024-09-21T09:58:44.094 INFO:tasks.rados.rados.0.smithi033.stdout:2315: done (5 left) 2024-09-21T09:58:44.094 INFO:tasks.rados.rados.0.smithi033.stdout:2316: done (4 left) 2024-09-21T09:58:44.094 INFO:tasks.rados.rados.0.smithi033.stdout:2317: done (3 left) 2024-09-21T09:58:44.094 INFO:tasks.rados.rados.0.smithi033.stdout:2318: done (2 left) 2024-09-21T09:58:44.095 INFO:tasks.rados.rados.0.smithi033.stdout:2319: done (1 left) 2024-09-21T09:58:44.095 INFO:tasks.rados.rados.0.smithi033.stdout:2321: done (0 left) 2024-09-21T09:58:44.095 INFO:tasks.rados.rados.0.smithi033.stdout:2322: delete oid 11 current snap is 215 2024-09-21T09:58:44.097 INFO:tasks.rados.rados.0.smithi033.stdout:2322: done (0 left) 2024-09-21T09:58:44.097 INFO:tasks.rados.rados.0.smithi033.stdout:2323: snap_create 2024-09-21T09:58:44.614 INFO:tasks.rados.rados.0.smithi033.stdout:2323: done (0 left) 2024-09-21T09:58:44.614 INFO:tasks.rados.rados.0.smithi033.stdout:2324: rmattr oid 43 current snap is 216 2024-09-21T09:58:44.614 INFO:tasks.rados.rados.0.smithi033.stdout:2325: read oid 29 snap -1 2024-09-21T09:58:44.614 INFO:tasks.rados.rados.0.smithi033.stdout:2325: expect (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:58:44.614 INFO:tasks.rados.rados.0.smithi033.stdout:2326: copy_from oid 33 from oid 14 current snap is 216 2024-09-21T09:58:44.614 INFO:tasks.rados.rados.0.smithi033.stdout:2327: copy_from oid 49 from oid 17 current snap is 216 2024-09-21T09:58:44.614 INFO:tasks.rados.rados.0.smithi033.stdout:2328: delete oid 10 current snap is 216 2024-09-21T09:58:44.616 INFO:tasks.rados.rados.0.smithi033.stdout:2327: finishing copy_from to smithi03385152-49 2024-09-21T09:58:44.616 INFO:tasks.rados.rados.0.smithi033.stdout:2327: got expected ENOENT (src dne) 2024-09-21T09:58:44.616 INFO:tasks.rados.rados.0.smithi033.stdout:2327: finishing copy_from racing read to smithi03385152-49 2024-09-21T09:58:44.621 INFO:tasks.rados.rados.0.smithi033.stdout:2327: done (4 left) 2024-09-21T09:58:44.621 INFO:tasks.rados.rados.0.smithi033.stdout:2328: done (3 left) 2024-09-21T09:58:44.621 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 216 2024-09-21T09:58:44.621 INFO:tasks.rados.rados.0.smithi033.stdout:2329: seq_num 745 ranges {417792=483328} 2024-09-21T09:58:44.626 INFO:tasks.rados.rados.0.smithi033.stdout:2329: writing smithi03385152-19 from 417792 to 901120 tid 1 2024-09-21T09:58:44.626 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 216 2024-09-21T09:58:44.626 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1036 (ObjNum 690 snap 192 seq_num 690) dirty exists 2024-09-21T09:58:44.626 INFO:tasks.rados.rados.0.smithi033.stdout:2330: seq_num 746 ranges {5382144=499712} 2024-09-21T09:58:44.659 INFO:tasks.rados.rados.0.smithi033.stdout:2330: writing smithi03385152-4 from 5382144 to 5881856 tid 1 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2324: done (4 left) 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2331: read oid 37 snap -1 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2331: expect (ObjNum 731 snap 210 seq_num 731) 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2332: copy_from oid 10 from oid 38 current snap is 216 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2329: finishing write tid 1 to smithi03385152-19 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2329: finishing write tid 2 to smithi03385152-19 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2329: finishing write tid 3 to smithi03385152-19 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1030 (ObjNum 745 snap 216 seq_num 745) dirty exists 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2329: left oid 19 (ObjNum 745 snap 216 seq_num 745) 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2326: finishing copy_from racing read to smithi03385152-33 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2326: finishing copy_from to smithi03385152-33 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1066 (ObjNum 666 snap 185 seq_num 666) dirty exists 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2326: done (5 left) 2024-09-21T09:58:44.660 INFO:tasks.rados.rados.0.smithi033.stdout:2329: done (4 left) 2024-09-21T09:58:44.661 INFO:tasks.rados.rados.0.smithi033.stdout:2333: read oid 19 snap -1 2024-09-21T09:58:44.661 INFO:tasks.rados.rados.0.smithi033.stdout:2333: expect (ObjNum 745 snap 216 seq_num 745) 2024-09-21T09:58:44.661 INFO:tasks.rados.rados.0.smithi033.stdout:2334: setattr oid 22 current snap is 216 2024-09-21T09:58:44.661 INFO:tasks.rados.rados.0.smithi033.stdout:2335: setattr oid 15 current snap is 216 2024-09-21T09:58:44.663 INFO:tasks.rados.rados.0.smithi033.stdout:2336: rmattr oid 30 current snap is 216 2024-09-21T09:58:44.663 INFO:tasks.rados.rados.0.smithi033.stdout:2336: done (7 left) 2024-09-21T09:58:44.664 INFO:tasks.rados.rados.0.smithi033.stdout:2337: read oid 36 snap -1 2024-09-21T09:58:44.664 INFO:tasks.rados.rados.0.smithi033.stdout:2337: expect (ObjNum 738 snap 214 seq_num 738) 2024-09-21T09:58:44.664 INFO:tasks.rados.rados.0.smithi033.stdout:2338: read oid 35 snap -1 2024-09-21T09:58:44.664 INFO:tasks.rados.rados.0.smithi033.stdout:2338: expect (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:58:44.664 INFO:tasks.rados.rados.0.smithi033.stdout:2339: rollback oid 44 current snap is 216 2024-09-21T09:58:44.664 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 44 to 202 2024-09-21T09:58:44.665 INFO:tasks.rados.rados.0.smithi033.stdout:2340: rmattr oid 8 current snap is 216 2024-09-21T09:58:44.665 INFO:tasks.rados.rados.0.smithi033.stdout:2340: done (10 left) 2024-09-21T09:58:44.665 INFO:tasks.rados.rados.0.smithi033.stdout:2341: copy_from oid 26 from oid 21 current snap is 216 2024-09-21T09:58:44.665 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 216 2024-09-21T09:58:44.665 INFO:tasks.rados.rados.0.smithi033.stdout:2342: seq_num 747 ranges {0=516096} 2024-09-21T09:58:44.666 INFO:tasks.rados.rados.0.smithi033.stdout:2342: writing smithi03385152-2 from 0 to 516096 tid 1 2024-09-21T09:58:44.666 INFO:tasks.rados.rados.0.smithi033.stdout:2343: snap_remove snap 211 2024-09-21T09:58:45.628 INFO:tasks.rados.rados.0.smithi033.stdout:2343: done (12 left) 2024-09-21T09:58:45.629 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 216 2024-09-21T09:58:45.629 INFO:tasks.rados.rados.0.smithi033.stdout:2344: seq_num 748 ranges {2629632=532480} 2024-09-21T09:58:45.647 INFO:tasks.rados.rados.0.smithi033.stdout:2344: writing smithi03385152-25 from 2629632 to 3162112 tid 1 2024-09-21T09:58:45.647 INFO:tasks.rados.rados.0.smithi033.stdout:2345: rollback oid 42 current snap is 216 2024-09-21T09:58:45.647 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 42 to 205 2024-09-21T09:58:45.648 INFO:tasks.rados.rados.0.smithi033.stdout:2331: expect (ObjNum 731 snap 210 seq_num 731) 2024-09-21T09:58:45.656 INFO:tasks.rados.rados.0.smithi033.stdout:2332: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:58:45.656 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 922 (ObjNum 684 snap 188 seq_num 684) dirty exists 2024-09-21T09:58:45.656 INFO:tasks.rados.rados.0.smithi033.stdout:2330: finishing write tid 1 to smithi03385152-4 2024-09-21T09:58:45.656 INFO:tasks.rados.rados.0.smithi033.stdout:2331: done (13 left) 2024-09-21T09:58:45.657 INFO:tasks.rados.rados.0.smithi033.stdout:2335: done (12 left) 2024-09-21T09:58:45.657 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 216 2024-09-21T09:58:45.657 INFO:tasks.rados.rados.0.smithi033.stdout:2330: finishing write tid 2 to smithi03385152-4 2024-09-21T09:58:45.657 INFO:tasks.rados.rados.0.smithi033.stdout:2346: seq_num 749 ranges {4300800=548864} 2024-09-21T09:58:45.683 INFO:tasks.rados.rados.0.smithi033.stdout:2346: writing smithi03385152-32 from 4300800 to 4849664 tid 1 2024-09-21T09:58:45.683 INFO:tasks.rados.rados.0.smithi033.stdout:2347: snap_create 2024-09-21T09:58:45.683 INFO:tasks.rados.rados.0.smithi033.stdout:2333: expect (ObjNum 745 snap 216 seq_num 745) 2024-09-21T09:58:45.714 INFO:tasks.rados.rados.0.smithi033.stdout:2332: finishing copy_from to smithi03385152-10 2024-09-21T09:58:45.714 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 772 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2339: finishing rollback tid 0 to smithi03385152-44 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2341: finishing copy_from racing read to smithi03385152-26 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2330: finishing write tid 3 to smithi03385152-4 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1046 (ObjNum 746 snap 216 seq_num 746) dirty exists 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2330: left oid 4 (ObjNum 746 snap 216 seq_num 746) 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2339: finishing rollback tid 1 to smithi03385152-44 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2339: finishing rollback tid 2 to smithi03385152-44 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1049 (ObjNum 608 snap 162 seq_num 608) dirty exists 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2342: finishing write tid 1 to smithi03385152-2 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2341: finishing copy_from to smithi03385152-26 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1473 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2342: finishing write tid 2 to smithi03385152-2 2024-09-21T09:58:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:2342: finishing write tid 3 to smithi03385152-2 2024-09-21T09:58:45.716 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 927 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T09:58:45.716 INFO:tasks.rados.rados.0.smithi033.stdout:2342: left oid 2 (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:58:45.716 INFO:tasks.rados.rados.0.smithi033.stdout:2325: expect (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:58:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:45 smithi033 ceph-mon[98546]: pgmap v373: 121 pgs: 3 peering, 30 active+undersized, 11 active+undersized+degraded, 77 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 727 KiB/s wr, 19 op/s; 85/534 objects degraded (15.918%); 2.0 MiB/s, 0 objects/s recovering 2024-09-21T09:58:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:45 smithi033 ceph-mon[98546]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:58:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:45 smithi033 ceph-mon[96568]: pgmap v373: 121 pgs: 3 peering, 30 active+undersized, 11 active+undersized+degraded, 77 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 727 KiB/s wr, 19 op/s; 85/534 objects degraded (15.918%); 2.0 MiB/s, 0 objects/s recovering 2024-09-21T09:58:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:45 smithi033 ceph-mon[96568]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:58:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:45 smithi105 ceph-mon[86649]: pgmap v373: 121 pgs: 3 peering, 30 active+undersized, 11 active+undersized+degraded, 77 active+clean; 276 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 727 KiB/s wr, 19 op/s; 85/534 objects degraded (15.918%); 2.0 MiB/s, 0 objects/s recovering 2024-09-21T09:58:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:45 smithi105 ceph-mon[86649]: osdmap e496: 8 total, 8 up, 8 in 2024-09-21T09:58:46.926 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1221 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:58:46.926 INFO:tasks.rados.rados.0.smithi033.stdout:2337: expect (ObjNum 738 snap 214 seq_num 738) 2024-09-21T09:58:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:46 smithi033 ceph-mon[98546]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:58:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:46 smithi033 ceph-mon[96568]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:58:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:46 smithi105 ceph-mon[86649]: osdmap e497: 8 total, 8 up, 8 in 2024-09-21T09:58:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:58:47.188 INFO:tasks.rados.rados.0.smithi033.stdout:2338: expect (ObjNum 680 snap 187 seq_num 680) 2024-09-21T09:58:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:47 smithi033 ceph-mon[96568]: pgmap v376: 121 pgs: 3 peering, 1 active+undersized, 1 active+undersized+degraded, 116 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.0 MiB/s wr, 17 op/s; 4/540 objects degraded (0.741%); 22 MiB/s, 4 objects/s recovering 2024-09-21T09:58:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:47 smithi033 ceph-mon[96568]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:58:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:47 smithi033 ceph-mon[98546]: pgmap v376: 121 pgs: 3 peering, 1 active+undersized, 1 active+undersized+degraded, 116 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.0 MiB/s wr, 17 op/s; 4/540 objects degraded (0.741%); 22 MiB/s, 4 objects/s recovering 2024-09-21T09:58:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:47 smithi033 ceph-mon[98546]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:58:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:47 smithi105 ceph-mon[86649]: pgmap v376: 121 pgs: 3 peering, 1 active+undersized, 1 active+undersized+degraded, 116 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 1.0 MiB/s wr, 17 op/s; 4/540 objects degraded (0.741%); 22 MiB/s, 4 objects/s recovering 2024-09-21T09:58:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:47 smithi105 ceph-mon[86649]: osdmap e498: 8 total, 8 up, 8 in 2024-09-21T09:58:48.397 INFO:tasks.rados.rados.0.smithi033.stdout:2345: finishing rollback tid 0 to smithi03385152-42 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2344: finishing write tid 1 to smithi03385152-25 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2345: finishing rollback tid 1 to smithi03385152-42 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2344: finishing write tid 2 to smithi03385152-25 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2345: finishing rollback tid 2 to smithi03385152-42 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 828 (ObjNum 683 snap 187 seq_num 683) dirty exists 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2344: finishing write tid 3 to smithi03385152-25 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 929 (ObjNum 748 snap 216 seq_num 748) dirty exists 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2344: left oid 25 (ObjNum 748 snap 216 seq_num 748) 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2325: done (13 left) 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2330: done (12 left) 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2332: done (11 left) 2024-09-21T09:58:48.398 INFO:tasks.rados.rados.0.smithi033.stdout:2333: done (10 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2334: done (9 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2337: done (8 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2338: done (7 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2339: done (6 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2341: done (5 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2342: done (4 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2344: done (3 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2345: done (2 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2347: done (1 left) 2024-09-21T09:58:48.399 INFO:tasks.rados.rados.0.smithi033.stdout:2348: snap_remove snap 208 2024-09-21T09:58:48.643 INFO:tasks.rados.rados.0.smithi033.stdout:2348: done (1 left) 2024-09-21T09:58:48.643 INFO:tasks.rados.rados.0.smithi033.stdout:2349: read oid 28 snap -1 2024-09-21T09:58:48.643 INFO:tasks.rados.rados.0.smithi033.stdout:2349: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:58:48.643 INFO:tasks.rados.rados.0.smithi033.stdout:2350: read oid 17 snap -1 2024-09-21T09:58:48.643 INFO:tasks.rados.rados.0.smithi033.stdout:2350: expect deleted 2024-09-21T09:58:48.643 INFO:tasks.rados.rados.0.smithi033.stdout:2351: snap_create 2024-09-21T09:58:48.643 INFO:tasks.rados.rados.0.smithi033.stdout:2346: finishing write tid 1 to smithi03385152-32 2024-09-21T09:58:48.643 INFO:tasks.rados.rados.0.smithi033.stdout:2346: finishing write tid 2 to smithi03385152-32 2024-09-21T09:58:48.644 INFO:tasks.rados.rados.0.smithi033.stdout:2346: finishing write tid 3 to smithi03385152-32 2024-09-21T09:58:48.644 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 209 (ObjNum 749 snap 216 seq_num 749) dirty exists 2024-09-21T09:58:48.644 INFO:tasks.rados.rados.0.smithi033.stdout:2346: left oid 32 (ObjNum 749 snap 216 seq_num 749) 2024-09-21T09:58:48.654 INFO:tasks.rados.rados.0.smithi033.stdout:2349: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:58:48.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:48 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 4/540 objects degraded (0.741%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:58:48.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:48 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 4/540 objects degraded (0.741%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:58:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:48 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 4/540 objects degraded (0.741%), 1 pg degraded (PG_DEGRADED) 2024-09-21T09:58:49.639 INFO:tasks.rados.rados.0.smithi033.stdout:2346: done (3 left) 2024-09-21T09:58:49.639 INFO:tasks.rados.rados.0.smithi033.stdout:2349: done (2 left) 2024-09-21T09:58:49.640 INFO:tasks.rados.rados.0.smithi033.stdout:2350: done (1 left) 2024-09-21T09:58:49.640 INFO:tasks.rados.rados.0.smithi033.stdout:2351: done (0 left) 2024-09-21T09:58:49.640 INFO:tasks.rados.rados.0.smithi033.stdout:2352: setattr oid 33 current snap is 218 2024-09-21T09:58:49.640 INFO:tasks.rados.rados.0.smithi033.stdout:2353: delete oid 35 current snap is 218 2024-09-21T09:58:49.651 INFO:tasks.rados.rados.0.smithi033.stdout:2353: done (1 left) 2024-09-21T09:58:49.652 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 218 2024-09-21T09:58:49.652 INFO:tasks.rados.rados.0.smithi033.stdout:2354: seq_num 751 ranges {4440064=581632} 2024-09-21T09:58:49.679 INFO:tasks.rados.rados.0.smithi033.stdout:2354: writing smithi03385152-49 from 4440064 to 5021696 tid 1 2024-09-21T09:58:49.679 INFO:tasks.rados.rados.0.smithi033.stdout:2355: snap_create 2024-09-21T09:58:49.679 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1076 (ObjNum 666 snap 185 seq_num 666) dirty exists 2024-09-21T09:58:49.683 INFO:tasks.rados.rados.0.smithi033.stdout:2354: finishing write tid 1 to smithi03385152-49 2024-09-21T09:58:49.683 INFO:tasks.rados.rados.0.smithi033.stdout:2354: finishing write tid 2 to smithi03385152-49 2024-09-21T09:58:49.684 INFO:tasks.rados.rados.0.smithi033.stdout:2354: finishing write tid 3 to smithi03385152-49 2024-09-21T09:58:49.684 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1475 (ObjNum 751 snap 218 seq_num 751) dirty exists 2024-09-21T09:58:49.685 INFO:tasks.rados.rados.0.smithi033.stdout:2354: left oid 49 (ObjNum 751 snap 218 seq_num 751) 2024-09-21T09:58:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:49 smithi033 ceph-mon[98546]: pgmap v378: 121 pgs: 121 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 881 KiB/s wr, 18 op/s; 18 MiB/s, 3 objects/s recovering 2024-09-21T09:58:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:49 smithi033 ceph-mon[98546]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/540 objects degraded (0.741%), 1 pg degraded) 2024-09-21T09:58:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:49 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T09:58:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:49 smithi033 ceph-mon[98546]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:58:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:49 smithi033 ceph-mon[96568]: pgmap v378: 121 pgs: 121 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 881 KiB/s wr, 18 op/s; 18 MiB/s, 3 objects/s recovering 2024-09-21T09:58:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:49 smithi033 ceph-mon[96568]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/540 objects degraded (0.741%), 1 pg degraded) 2024-09-21T09:58:49.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:49 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T09:58:49.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:49 smithi033 ceph-mon[96568]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:58:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:49 smithi105 ceph-mon[86649]: pgmap v378: 121 pgs: 121 active+clean; 270 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 881 KiB/s wr, 18 op/s; 18 MiB/s, 3 objects/s recovering 2024-09-21T09:58:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:49 smithi105 ceph-mon[86649]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/540 objects degraded (0.741%), 1 pg degraded) 2024-09-21T09:58:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:49 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T09:58:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:49 smithi105 ceph-mon[86649]: osdmap e499: 8 total, 8 up, 8 in 2024-09-21T09:58:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:58:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:58:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:58:50.651 INFO:tasks.rados.rados.0.smithi033.stdout:2352: done (2 left) 2024-09-21T09:58:50.651 INFO:tasks.rados.rados.0.smithi033.stdout:2354: done (1 left) 2024-09-21T09:58:50.651 INFO:tasks.rados.rados.0.smithi033.stdout:2355: done (0 left) 2024-09-21T09:58:50.651 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 219 2024-09-21T09:58:50.651 INFO:tasks.rados.rados.0.smithi033.stdout:2356: seq_num 752 ranges {4849664=598016} 2024-09-21T09:58:50.680 INFO:tasks.rados.rados.0.smithi033.stdout:2356: writing smithi03385152-32 from 4849664 to 5447680 tid 1 2024-09-21T09:58:50.680 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 219 2024-09-21T09:58:50.680 INFO:tasks.rados.rados.0.smithi033.stdout:2357: seq_num 753 ranges {2629632=614400} 2024-09-21T09:58:50.700 INFO:tasks.rados.rados.0.smithi033.stdout:2357: writing smithi03385152-12 from 2629632 to 3244032 tid 1 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:2358: rmattr oid 2 current snap is 219 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:2358: done (2 left) 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:2359: rollback oid 46 current snap is 219 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 46 to 217 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:2356: finishing write tid 1 to smithi03385152-32 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:2356: finishing write tid 2 to smithi03385152-32 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:2356: finishing write tid 3 to smithi03385152-32 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 214 (ObjNum 752 snap 219 seq_num 752) dirty exists 2024-09-21T09:58:50.701 INFO:tasks.rados.rados.0.smithi033.stdout:2356: left oid 32 (ObjNum 752 snap 219 seq_num 752) 2024-09-21T09:58:50.702 INFO:tasks.rados.rados.0.smithi033.stdout:2356: done (2 left) 2024-09-21T09:58:50.702 INFO:tasks.rados.rados.0.smithi033.stdout:2360: delete oid 14 current snap is 219 2024-09-21T09:58:50.702 INFO:tasks.rados.rados.0.smithi033.stdout:2359: finishing rollback tid 0 to smithi03385152-46 2024-09-21T09:58:50.703 INFO:tasks.rados.rados.0.smithi033.stdout:2357: finishing write tid 1 to smithi03385152-12 2024-09-21T09:58:50.703 INFO:tasks.rados.rados.0.smithi033.stdout:2357: finishing write tid 2 to smithi03385152-12 2024-09-21T09:58:50.704 INFO:tasks.rados.rados.0.smithi033.stdout:2359: finishing rollback tid 1 to smithi03385152-46 2024-09-21T09:58:50.705 INFO:tasks.rados.rados.0.smithi033.stdout:2359: finishing rollback tid 2 to smithi03385152-46 2024-09-21T09:58:50.705 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1239 (ObjNum 667 snap 185 seq_num 667) dirty exists 2024-09-21T09:58:50.705 INFO:tasks.rados.rados.0.smithi033.stdout:2357: finishing write tid 3 to smithi03385152-12 2024-09-21T09:58:50.705 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1477 (ObjNum 753 snap 219 seq_num 753) dirty exists 2024-09-21T09:58:50.705 INFO:tasks.rados.rados.0.smithi033.stdout:2357: left oid 12 (ObjNum 753 snap 219 seq_num 753) 2024-09-21T09:58:50.706 INFO:tasks.rados.rados.0.smithi033.stdout:2357: done (2 left) 2024-09-21T09:58:50.706 INFO:tasks.rados.rados.0.smithi033.stdout:2359: done (1 left) 2024-09-21T09:58:50.706 INFO:tasks.rados.rados.0.smithi033.stdout:2360: done (0 left) 2024-09-21T09:58:50.706 INFO:tasks.rados.rados.0.smithi033.stdout:2361: setattr oid 7 current snap is 219 2024-09-21T09:58:50.706 INFO:tasks.rados.rados.0.smithi033.stdout:2362: rmattr oid 5 current snap is 219 2024-09-21T09:58:50.706 INFO:tasks.rados.rados.0.smithi033.stdout:2362: done (1 left) 2024-09-21T09:58:50.707 INFO:tasks.rados.rados.0.smithi033.stdout:2363: read oid 43 snap -1 2024-09-21T09:58:50.707 INFO:tasks.rados.rados.0.smithi033.stdout:2363: expect (ObjNum 690 snap 192 seq_num 690) 2024-09-21T09:58:50.707 INFO:tasks.rados.rados.0.smithi033.stdout:2364: read oid 46 snap -1 2024-09-21T09:58:50.707 INFO:tasks.rados.rados.0.smithi033.stdout:2364: expect (ObjNum 667 snap 185 seq_num 667) 2024-09-21T09:58:50.707 INFO:tasks.rados.rados.0.smithi033.stdout:2365: copy_from oid 10 from oid 41 current snap is 219 2024-09-21T09:58:50.707 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 219 2024-09-21T09:58:50.707 INFO:tasks.rados.rados.0.smithi033.stdout:2366: seq_num 755 ranges {1392640=647168} 2024-09-21T09:58:50.719 INFO:tasks.rados.rados.0.smithi033.stdout:2366: writing smithi03385152-34 from 1392640 to 2039808 tid 1 2024-09-21T09:58:50.719 INFO:tasks.rados.rados.0.smithi033.stdout:2367: delete oid 30 current snap is 219 2024-09-21T09:58:50.720 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 1568 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:58:50.720 INFO:tasks.rados.rados.0.smithi033.stdout:2364: expect (ObjNum 667 snap 185 seq_num 667) 2024-09-21T09:58:50.777 INFO:tasks.rados.rados.0.smithi033.stdout:2365: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:58:50.777 INFO:tasks.rados.rados.0.smithi033.stdout:2365: finishing copy_from to smithi03385152-10 2024-09-21T09:58:50.777 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 770 (ObjNum 692 snap 192 seq_num 692) dirty exists 2024-09-21T09:58:50.777 INFO:tasks.rados.rados.0.smithi033.stdout:2363: expect (ObjNum 690 snap 192 seq_num 690) 2024-09-21T09:58:50.837 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:58:50.921 INFO:tasks.rados.rados.0.smithi033.stdout:2366: finishing write tid 1 to smithi03385152-34 2024-09-21T09:58:50.922 INFO:tasks.rados.rados.0.smithi033.stdout:2366: finishing write tid 2 to smithi03385152-34 2024-09-21T09:58:50.922 INFO:tasks.rados.rados.0.smithi033.stdout:2366: finishing write tid 3 to smithi03385152-34 2024-09-21T09:58:50.922 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1244 (ObjNum 755 snap 219 seq_num 755) dirty exists 2024-09-21T09:58:50.922 INFO:tasks.rados.rados.0.smithi033.stdout:2366: left oid 34 (ObjNum 755 snap 219 seq_num 755) 2024-09-21T09:58:50.922 INFO:tasks.rados.rados.0.smithi033.stdout:2361: done (5 left) 2024-09-21T09:58:50.922 INFO:tasks.rados.rados.0.smithi033.stdout:2363: done (4 left) 2024-09-21T09:58:50.922 INFO:tasks.rados.rados.0.smithi033.stdout:2364: done (3 left) 2024-09-21T09:58:50.922 INFO:tasks.rados.rados.0.smithi033.stdout:2365: done (2 left) 2024-09-21T09:58:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:2366: done (1 left) 2024-09-21T09:58:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:2367: done (0 left) 2024-09-21T09:58:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:2368: read oid 25 snap -1 2024-09-21T09:58:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:2368: expect (ObjNum 748 snap 216 seq_num 748) 2024-09-21T09:58:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 219 2024-09-21T09:58:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:2369: seq_num 757 ranges {0=688128} 2024-09-21T09:58:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:2369: writing smithi03385152-20 from 0 to 688128 tid 1 2024-09-21T09:58:50.923 INFO:tasks.rados.rados.0.smithi033.stdout:2370: delete oid 33 current snap is 219 2024-09-21T09:58:50.928 INFO:tasks.rados.rados.0.smithi033.stdout:2370: done (2 left) 2024-09-21T09:58:50.928 INFO:tasks.rados.rados.0.smithi033.stdout:2371: delete oid 14 current snap is 219 2024-09-21T09:58:50.929 INFO:tasks.rados.rados.0.smithi033.stdout:2369: finishing write tid 1 to smithi03385152-20 2024-09-21T09:58:50.929 INFO:tasks.rados.rados.0.smithi033.stdout:2369: finishing write tid 2 to smithi03385152-20 2024-09-21T09:58:50.929 INFO:tasks.rados.rados.0.smithi033.stdout:2371: done (2 left) 2024-09-21T09:58:50.929 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 219 2024-09-21T09:58:50.929 INFO:tasks.rados.rados.0.smithi033.stdout:2372: seq_num 760 ranges {2662400=737280} 2024-09-21T09:58:50.951 INFO:tasks.rados.rados.0.smithi033.stdout:2372: writing smithi03385152-6 from 2662400 to 3399680 tid 1 2024-09-21T09:58:50.951 INFO:tasks.rados.rados.0.smithi033.stdout:2373: setattr oid 3 current snap is 219 2024-09-21T09:58:50.951 INFO:tasks.rados.rados.0.smithi033.stdout:2369: finishing write tid 3 to smithi03385152-20 2024-09-21T09:58:50.952 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1476 (ObjNum 757 snap 219 seq_num 757) dirty exists 2024-09-21T09:58:50.952 INFO:tasks.rados.rados.0.smithi033.stdout:2369: left oid 20 (ObjNum 757 snap 219 seq_num 757) 2024-09-21T09:58:50.952 INFO:tasks.rados.rados.0.smithi033.stdout:2368: expect (ObjNum 748 snap 216 seq_num 748) 2024-09-21T09:58:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:50 smithi033 ceph-mon[98546]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:58:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:50 smithi033 ceph-mon[96568]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:58:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:50 smithi105 ceph-mon[86649]: osdmap e500: 8 total, 8 up, 8 in 2024-09-21T09:58:51.092 INFO:tasks.rados.rados.0.smithi033.stdout:2372: finishing write tid 1 to smithi03385152-6 2024-09-21T09:58:51.092 INFO:tasks.rados.rados.0.smithi033.stdout:2372: finishing write tid 2 to smithi03385152-6 2024-09-21T09:58:51.092 INFO:tasks.rados.rados.0.smithi033.stdout:2372: finishing write tid 3 to smithi03385152-6 2024-09-21T09:58:51.092 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1079 (ObjNum 760 snap 219 seq_num 760) dirty exists 2024-09-21T09:58:51.092 INFO:tasks.rados.rados.0.smithi033.stdout:2372: left oid 6 (ObjNum 760 snap 219 seq_num 760) 2024-09-21T09:58:51.092 INFO:tasks.rados.rados.0.smithi033.stdout:2368: done (3 left) 2024-09-21T09:58:51.092 INFO:tasks.rados.rados.0.smithi033.stdout:2369: done (2 left) 2024-09-21T09:58:51.092 INFO:tasks.rados.rados.0.smithi033.stdout:2372: done (1 left) 2024-09-21T09:58:51.093 INFO:tasks.rados.rados.0.smithi033.stdout:2374: snap_remove snap 190 2024-09-21T09:58:51.441 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:58:51.441 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (5m) 80s ago 10m 19.2M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:58:51.441 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (5m) 12s ago 9m 87.5M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:58:51.441 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (11m) 80s ago 11m 16.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (11m) 12s ago 11m 21.6M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (4m) 12s ago 15m 455M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (5m) 80s ago 16m 524M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (4m) 80s ago 16m 56.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (3m) 12s ago 15m 47.5M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (4m) 80s ago 15m 47.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (5m) 80s ago 9m 11.8M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (5m) 12s ago 9m 10.8M - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (3m) 80s ago 14m 168M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (2m) 80s ago 14m 126M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (118s) 80s ago 14m 66.9M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 76dbf29565ba 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (84s) 80s ago 13m 11.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b85b61817088 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (51s) 12s ago 13m 92.1M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 01f4f66dbe72 2024-09-21T09:58:51.442 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (17s) 12s ago 12m 11.4M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4ff1257a8be1 2024-09-21T09:58:51.443 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (12m) 12s ago 12m 204M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 37fd995b47ba 2024-09-21T09:58:51.443 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (12m) 12s ago 12m 201M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:58:51.443 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (4m) 12s ago 10m 51.5M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:58:51.656 INFO:tasks.rados.rados.0.smithi033.stdout:2374: done (1 left) 2024-09-21T09:58:51.656 INFO:tasks.rados.rados.0.smithi033.stdout:2375: delete oid 7 current snap is 219 2024-09-21T09:58:51.656 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1247 (ObjNum 450 snap 120 seq_num 450) dirty exists 2024-09-21T09:58:51.672 INFO:tasks.rados.rados.0.smithi033.stdout:2373: done (1 left) 2024-09-21T09:58:51.672 INFO:tasks.rados.rados.0.smithi033.stdout:2375: done (0 left) 2024-09-21T09:58:51.672 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 219 2024-09-21T09:58:51.672 INFO:tasks.rados.rados.0.smithi033.stdout:2376: seq_num 762 ranges {385024=770048} 2024-09-21T09:58:51.680 INFO:tasks.rados.rados.0.smithi033.stdout:2376: writing smithi03385152-21 from 385024 to 1155072 tid 1 2024-09-21T09:58:51.680 INFO:tasks.rados.rados.0.smithi033.stdout:2377: rollback oid 24 current snap is 219 2024-09-21T09:58:51.680 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 196 2024-09-21T09:58:51.680 INFO:tasks.rados.rados.0.smithi033.stdout:2378: copy_from oid 44 from oid 15 current snap is 219 2024-09-21T09:58:51.680 INFO:tasks.rados.rados.0.smithi033.stdout:2379: rmattr oid 11 current snap is 219 2024-09-21T09:58:51.680 INFO:tasks.rados.rados.0.smithi033.stdout:2379: done (3 left) 2024-09-21T09:58:51.680 INFO:tasks.rados.rados.0.smithi033.stdout:2380: rmattr oid 47 current snap is 219 2024-09-21T09:58:51.681 INFO:tasks.rados.rados.0.smithi033.stdout:2380: done (3 left) 2024-09-21T09:58:51.681 INFO:tasks.rados.rados.0.smithi033.stdout:2381: delete oid 39 current snap is 219 2024-09-21T09:58:51.681 INFO:tasks.rados.rados.0.smithi033.stdout:2377: finishing rollback tid 1 to smithi03385152-24 2024-09-21T09:58:51.681 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T09:58:51.683 INFO:tasks.rados.rados.0.smithi033.stdout:2377: done (3 left) 2024-09-21T09:58:51.683 INFO:tasks.rados.rados.0.smithi033.stdout:2381: done (2 left) 2024-09-21T09:58:51.684 INFO:tasks.rados.rados.0.smithi033.stdout:2382: read oid 2 snap -1 2024-09-21T09:58:51.684 INFO:tasks.rados.rados.0.smithi033.stdout:2382: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:58:51.684 INFO:tasks.rados.rados.0.smithi033.stdout:2383: read oid 10 snap -1 2024-09-21T09:58:51.684 INFO:tasks.rados.rados.0.smithi033.stdout:2383: expect (ObjNum 692 snap 192 seq_num 692) 2024-09-21T09:58:51.684 INFO:tasks.rados.rados.0.smithi033.stdout:2384: read oid 47 snap -1 2024-09-21T09:58:51.684 INFO:tasks.rados.rados.0.smithi033.stdout:2384: expect deleted 2024-09-21T09:58:51.684 INFO:tasks.rados.rados.0.smithi033.stdout:2385: delete oid 16 current snap is 219 2024-09-21T09:58:51.685 INFO:tasks.rados.rados.0.smithi033.stdout:2376: finishing write tid 1 to smithi03385152-21 2024-09-21T09:58:51.685 INFO:tasks.rados.rados.0.smithi033.stdout:2376: finishing write tid 2 to smithi03385152-21 2024-09-21T09:58:51.686 INFO:tasks.rados.rados.0.smithi033.stdout:2385: done (5 left) 2024-09-21T09:58:51.686 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 219 2024-09-21T09:58:51.686 INFO:tasks.rados.rados.0.smithi033.stdout:2386: seq_num 765 ranges {0=417792,417792=401408} 2024-09-21T09:58:51.688 INFO:tasks.rados.rados.0.smithi033.stdout:2386: writing smithi03385152-7 from 0 to 417792 tid 1 2024-09-21T09:58:51.691 INFO:tasks.rados.rados.0.smithi033.stdout:2386: writing smithi03385152-7 from 417792 to 819200 tid 2 2024-09-21T09:58:51.691 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 219 2024-09-21T09:58:51.692 INFO:tasks.rados.rados.0.smithi033.stdout:2376: finishing write tid 3 to smithi03385152-21 2024-09-21T09:58:51.692 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1478 (ObjNum 762 snap 219 seq_num 762) dirty exists 2024-09-21T09:58:51.692 INFO:tasks.rados.rados.0.smithi033.stdout:2376: left oid 21 (ObjNum 762 snap 219 seq_num 762) 2024-09-21T09:58:51.692 INFO:tasks.rados.rados.0.smithi033.stdout:2382: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:2378: finishing copy_from racing read to smithi03385152-44 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:2386: finishing write tid 1 to smithi03385152-7 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:2386: finishing write tid 2 to smithi03385152-7 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:2386: finishing write tid 3 to smithi03385152-7 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:2378: finishing copy_from to smithi03385152-44 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 922 (ObjNum 684 snap 188 seq_num 684) dirty exists 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:2386: finishing write tid 4 to smithi03385152-7 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 1589 (ObjNum 765 snap 219 seq_num 765) dirty exists 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:2386: left oid 7 (ObjNum 765 snap 219 seq_num 765) 2024-09-21T09:58:51.707 INFO:tasks.rados.rados.0.smithi033.stdout:2383: expect (ObjNum 692 snap 192 seq_num 692) 2024-09-21T09:58:51.746 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:58:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:58:51] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.51.0" 2024-09-21T09:58:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:51 smithi033 ceph-mon[98546]: pgmap v381: 121 pgs: 121 active+clean; 271 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 709 KiB/s rd, 717 KiB/s wr, 11 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-21T09:58:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:51 smithi033 ceph-mon[98546]: osdmap e501: 8 total, 8 up, 8 in 2024-09-21T09:58:51.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:51 smithi033 ceph-mon[98546]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:51 smithi033 ceph-mon[96568]: pgmap v381: 121 pgs: 121 active+clean; 271 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 709 KiB/s rd, 717 KiB/s wr, 11 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-21T09:58:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:51 smithi033 ceph-mon[96568]: osdmap e501: 8 total, 8 up, 8 in 2024-09-21T09:58:51.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:51 smithi033 ceph-mon[96568]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:51.815 INFO:tasks.rados.rados.0.smithi033.stdout:2387: seq_num 766 ranges {0=434176,434176=401408} 2024-09-21T09:58:51.817 INFO:tasks.rados.rados.0.smithi033.stdout:2387: writing smithi03385152-33 from 0 to 434176 tid 1 2024-09-21T09:58:51.819 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:58:51.819 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:58:51.819 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:58:51.819 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2, 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 6 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:58:51.820 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 4, 2024-09-21T09:58:51.821 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 11 2024-09-21T09:58:51.821 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:58:51.821 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:58:51.821 INFO:tasks.rados.rados.0.smithi033.stdout:2387: writing smithi03385152-33 from 434176 to 835584 tid 2 2024-09-21T09:58:51.821 INFO:tasks.rados.rados.0.smithi033.stdout:2376: done (6 left) 2024-09-21T09:58:51.821 INFO:tasks.rados.rados.0.smithi033.stdout:2378: done (5 left) 2024-09-21T09:58:51.821 INFO:tasks.rados.rados.0.smithi033.stdout:2382: done (4 left) 2024-09-21T09:58:51.822 INFO:tasks.rados.rados.0.smithi033.stdout:2383: done (3 left) 2024-09-21T09:58:51.822 INFO:tasks.rados.rados.0.smithi033.stdout:2384: done (2 left) 2024-09-21T09:58:51.822 INFO:tasks.rados.rados.0.smithi033.stdout:2386: done (1 left) 2024-09-21T09:58:51.822 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 219 2024-09-21T09:58:51.822 INFO:tasks.rados.rados.0.smithi033.stdout:2388: seq_num 767 ranges {0=450560,450560=401408} 2024-09-21T09:58:51.822 INFO:tasks.rados.rados.0.smithi033.stdout:2388: writing smithi03385152-18 from 0 to 450560 tid 1 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2388: writing smithi03385152-18 from 450560 to 851968 tid 2 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2389: read oid 17 snap -1 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2389: expect deleted 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2390: delete oid 44 current snap is 219 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2387: finishing write tid 1 to smithi03385152-33 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2387: finishing write tid 2 to smithi03385152-33 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2387: finishing write tid 3 to smithi03385152-33 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2387: finishing write tid 4 to smithi03385152-33 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1082 (ObjNum 766 snap 219 seq_num 766) dirty exists 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2387: left oid 33 (ObjNum 766 snap 219 seq_num 766) 2024-09-21T09:58:51.826 INFO:tasks.rados.rados.0.smithi033.stdout:2388: finishing write tid 1 to smithi03385152-18 2024-09-21T09:58:51.827 INFO:tasks.rados.rados.0.smithi033.stdout:2387: done (3 left) 2024-09-21T09:58:51.827 INFO:tasks.rados.rados.0.smithi033.stdout:2389: done (2 left) 2024-09-21T09:58:51.827 INFO:tasks.rados.rados.0.smithi033.stdout:2390: done (1 left) 2024-09-21T09:58:51.827 INFO:tasks.rados.rados.0.smithi033.stdout:2391: rmattr oid 16 current snap is 219 2024-09-21T09:58:51.827 INFO:tasks.rados.rados.0.smithi033.stdout:2391: done (1 left) 2024-09-21T09:58:51.828 INFO:tasks.rados.rados.0.smithi033.stdout:2392: snap_remove snap 212 2024-09-21T09:58:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:51 smithi105 ceph-mon[86649]: pgmap v381: 121 pgs: 121 active+clean; 271 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 709 KiB/s rd, 717 KiB/s wr, 11 op/s; 11 MiB/s, 1 objects/s recovering 2024-09-21T09:58:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:51 smithi105 ceph-mon[86649]: osdmap e501: 8 total, 8 up, 8 in 2024-09-21T09:58:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:51 smithi105 ceph-mon[86649]: from='client.34428 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:52.121 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:58:52.121 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:58:52.121 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "11/20 daemons upgraded", 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:58:52.122 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:58:52.668 INFO:tasks.rados.rados.0.smithi033.stdout:2392: done (1 left) 2024-09-21T09:58:52.669 INFO:tasks.rados.rados.0.smithi033.stdout:2393: snap_create 2024-09-21T09:58:52.669 INFO:tasks.rados.rados.0.smithi033.stdout:2388: finishing write tid 2 to smithi03385152-18 2024-09-21T09:58:52.669 INFO:tasks.rados.rados.0.smithi033.stdout:2388: finishing write tid 3 to smithi03385152-18 2024-09-21T09:58:52.669 INFO:tasks.rados.rados.0.smithi033.stdout:2388: finishing write tid 4 to smithi03385152-18 2024-09-21T09:58:52.669 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1006 (ObjNum 767 snap 219 seq_num 767) dirty exists 2024-09-21T09:58:52.669 INFO:tasks.rados.rados.0.smithi033.stdout:2388: left oid 18 (ObjNum 767 snap 219 seq_num 767) 2024-09-21T09:58:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:52 smithi033 ceph-mon[96568]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:52 smithi033 ceph-mon[96568]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:52 smithi033 ceph-mon[96568]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T09:58:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:52 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/286359076' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:52 smithi033 ceph-mon[98546]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:52.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:52 smithi033 ceph-mon[98546]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:52.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:52 smithi033 ceph-mon[98546]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T09:58:52.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:52 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/286359076' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:52 smithi105 ceph-mon[86649]: from='client.44348 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:52 smithi105 ceph-mon[86649]: from='client.34440 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:52 smithi105 ceph-mon[86649]: osdmap e502: 8 total, 8 up, 8 in 2024-09-21T09:58:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:52 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/286359076' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:58:53.492 INFO:tasks.rados.rados.0.smithi033.stdout:2388: done (1 left) 2024-09-21T09:58:53.493 INFO:tasks.rados.rados.0.smithi033.stdout:2393: done (0 left) 2024-09-21T09:58:53.493 INFO:tasks.rados.rados.0.smithi033.stdout:2394: rollback oid 30 current snap is 220 2024-09-21T09:58:53.493 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 30 to 204 2024-09-21T09:58:53.493 INFO:tasks.rados.rados.0.smithi033.stdout:2395: rollback oid 21 current snap is 220 2024-09-21T09:58:53.493 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 204 2024-09-21T09:58:53.493 INFO:tasks.rados.rados.0.smithi033.stdout:2396: delete oid 10 current snap is 220 2024-09-21T09:58:53.500 INFO:tasks.rados.rados.0.smithi033.stdout:2396: done (2 left) 2024-09-21T09:58:53.500 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 220 2024-09-21T09:58:53.500 INFO:tasks.rados.rados.0.smithi033.stdout:2397: seq_num 770 ranges {1982464=499712,2482176=401408} 2024-09-21T09:58:53.514 INFO:tasks.rados.rados.0.smithi033.stdout:2397: writing smithi03385152-48 from 1982464 to 2482176 tid 1 2024-09-21T09:58:53.516 INFO:tasks.rados.rados.0.smithi033.stdout:2397: writing smithi03385152-48 from 2482176 to 2883584 tid 2 2024-09-21T09:58:53.516 INFO:tasks.rados.rados.0.smithi033.stdout:2398: snap_create 2024-09-21T09:58:53.517 INFO:tasks.rados.rados.0.smithi033.stdout:2395: finishing rollback tid 0 to smithi03385152-21 2024-09-21T09:58:53.517 INFO:tasks.rados.rados.0.smithi033.stdout:2394: finishing rollback tid 1 to smithi03385152-30 2024-09-21T09:58:53.517 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 1488 (ObjNum 129 snap 0 seq_num 91616936) dirty dne 2024-09-21T09:58:53.517 INFO:tasks.rados.rados.0.smithi033.stdout:2395: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:58:53.517 INFO:tasks.rados.rados.0.smithi033.stdout:2395: finishing rollback tid 2 to smithi03385152-21 2024-09-21T09:58:53.517 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1481 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:58:53.520 INFO:tasks.rados.rados.0.smithi033.stdout:2397: finishing write tid 1 to smithi03385152-48 2024-09-21T09:58:53.520 INFO:tasks.rados.rados.0.smithi033.stdout:2397: finishing write tid 2 to smithi03385152-48 2024-09-21T09:58:53.520 INFO:tasks.rados.rados.0.smithi033.stdout:2397: finishing write tid 3 to smithi03385152-48 2024-09-21T09:58:53.522 INFO:tasks.rados.rados.0.smithi033.stdout:2397: finishing write tid 4 to smithi03385152-48 2024-09-21T09:58:53.522 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 1271 (ObjNum 770 snap 220 seq_num 770) dirty exists 2024-09-21T09:58:53.522 INFO:tasks.rados.rados.0.smithi033.stdout:2397: left oid 48 (ObjNum 770 snap 220 seq_num 770) 2024-09-21T09:58:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:53 smithi033 ceph-mon[96568]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:53 smithi033 ceph-mon[96568]: pgmap v384: 121 pgs: 121 active+clean; 274 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 854 KiB/s wr, 15 op/s 2024-09-21T09:58:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:53 smithi033 ceph-mon[96568]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T09:58:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:53 smithi033 ceph-mon[96568]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T09:58:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:53 smithi033 ceph-mon[98546]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:53 smithi033 ceph-mon[98546]: pgmap v384: 121 pgs: 121 active+clean; 274 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 854 KiB/s wr, 15 op/s 2024-09-21T09:58:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:53 smithi033 ceph-mon[98546]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T09:58:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:53 smithi033 ceph-mon[98546]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T09:58:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:53 smithi105 ceph-mon[86649]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:58:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:53 smithi105 ceph-mon[86649]: pgmap v384: 121 pgs: 121 active+clean; 274 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 854 KiB/s wr, 15 op/s 2024-09-21T09:58:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:53 smithi105 ceph-mon[86649]: osdmap e503: 8 total, 8 up, 8 in 2024-09-21T09:58:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:53 smithi105 ceph-mon[86649]: osdmap e504: 8 total, 8 up, 8 in 2024-09-21T09:58:54.495 INFO:tasks.rados.rados.0.smithi033.stdout:2394: done (3 left) 2024-09-21T09:58:54.495 INFO:tasks.rados.rados.0.smithi033.stdout:2395: done (2 left) 2024-09-21T09:58:54.495 INFO:tasks.rados.rados.0.smithi033.stdout:2397: done (1 left) 2024-09-21T09:58:54.496 INFO:tasks.rados.rados.0.smithi033.stdout:2398: done (0 left) 2024-09-21T09:58:54.496 INFO:tasks.rados.rados.0.smithi033.stdout:2399: rmattr oid 36 current snap is 221 2024-09-21T09:58:54.496 INFO:tasks.rados.rados.0.smithi033.stdout:2399: done (0 left) 2024-09-21T09:58:54.496 INFO:tasks.rados.rados.0.smithi033.stdout:2400: delete oid 24 current snap is 221 2024-09-21T09:58:54.503 INFO:tasks.rados.rados.0.smithi033.stdout:2400: done (0 left) 2024-09-21T09:58:54.503 INFO:tasks.rados.rados.0.smithi033.stdout:2401: read oid 46 snap -1 2024-09-21T09:58:54.503 INFO:tasks.rados.rados.0.smithi033.stdout:2401: expect (ObjNum 667 snap 185 seq_num 667) 2024-09-21T09:58:54.503 INFO:tasks.rados.rados.0.smithi033.stdout:2402: setattr oid 5 current snap is 221 2024-09-21T09:58:54.504 INFO:tasks.rados.rados.0.smithi033.stdout:2403: copy_from oid 6 from oid 7 current snap is 221 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2404: read oid 48 snap -1 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2404: expect (ObjNum 770 snap 220 seq_num 770) 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2405: rmattr oid 34 current snap is 221 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2405: done (4 left) 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2406: read oid 18 snap -1 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2406: expect (ObjNum 767 snap 219 seq_num 767) 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2407: delete oid 19 current snap is 221 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2407: done (5 left) 2024-09-21T09:58:54.505 INFO:tasks.rados.rados.0.smithi033.stdout:2408: snap_create 2024-09-21T09:58:54.506 INFO:tasks.rados.rados.0.smithi033.stdout:2406: expect (ObjNum 767 snap 219 seq_num 767) 2024-09-21T09:58:54.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 1011 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:58:54.529 INFO:tasks.rados.rados.0.smithi033.stdout:2401: expect (ObjNum 667 snap 185 seq_num 667) 2024-09-21T09:58:54.572 INFO:tasks.rados.rados.0.smithi033.stdout:2403: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:58:54.572 INFO:tasks.rados.rados.0.smithi033.stdout:2403: finishing copy_from to smithi03385152-6 2024-09-21T09:58:54.573 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1589 (ObjNum 765 snap 219 seq_num 765) dirty exists 2024-09-21T09:58:54.573 INFO:tasks.rados.rados.0.smithi033.stdout:2404: expect (ObjNum 770 snap 220 seq_num 770) 2024-09-21T09:58:55.501 INFO:tasks.rados.rados.0.smithi033.stdout:2401: done (5 left) 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2402: done (4 left) 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2403: done (3 left) 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2404: done (2 left) 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2406: done (1 left) 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2408: done (0 left) 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2409: rollback oid 27 current snap is 222 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 27 to 209 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2410: read oid 11 snap 205 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2410: expect (ObjNum 663 snap 184 seq_num 663) 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2411: rollback oid 15 current snap is 222 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 15 to 199 2024-09-21T09:58:55.502 INFO:tasks.rados.rados.0.smithi033.stdout:2412: rollback oid 21 current snap is 222 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 216 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:2413: rollback oid 42 current snap is 222 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 42 to 173 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:2414: read oid 50 snap -1 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:2414: expect deleted 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:2415: rmattr oid 41 current snap is 222 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:2415: done (6 left) 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:2416: read oid 6 snap -1 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:2416: expect (ObjNum 765 snap 219 seq_num 765) 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 222 2024-09-21T09:58:55.503 INFO:tasks.rados.rados.0.smithi033.stdout:2417: seq_num 773 ranges {3309568=548864,3858432=401408} 2024-09-21T09:58:55.524 INFO:tasks.rados.rados.0.smithi033.stdout:2417: writing smithi03385152-41 from 3309568 to 3858432 tid 1 2024-09-21T09:58:55.529 INFO:tasks.rados.rados.0.smithi033.stdout:2417: writing smithi03385152-41 from 3858432 to 4259840 tid 2 2024-09-21T09:58:55.529 INFO:tasks.rados.rados.0.smithi033.stdout:2418: read oid 48 snap 214 2024-09-21T09:58:55.529 INFO:tasks.rados.rados.0.smithi033.stdout:2418: expect (ObjNum 691 snap 192 seq_num 691) 2024-09-21T09:58:55.529 INFO:tasks.rados.rados.0.smithi033.stdout:2419: snap_remove snap 217 2024-09-21T09:58:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:55 smithi033 ceph-mon[98546]: pgmap v387: 121 pgs: 121 active+clean; 277 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-21T09:58:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:55 smithi033 ceph-mon[98546]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T09:58:55.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:55 smithi033 ceph-mon[96568]: pgmap v387: 121 pgs: 121 active+clean; 277 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-21T09:58:55.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:55 smithi033 ceph-mon[96568]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T09:58:55.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:55 smithi105 ceph-mon[86649]: pgmap v387: 121 pgs: 121 active+clean; 277 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 1.7 MiB/s wr, 13 op/s 2024-09-21T09:58:55.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:55 smithi105 ceph-mon[86649]: osdmap e505: 8 total, 8 up, 8 in 2024-09-21T09:58:56.507 INFO:tasks.rados.rados.0.smithi033.stdout:2419: done (9 left) 2024-09-21T09:58:56.507 INFO:tasks.rados.rados.0.smithi033.stdout:2420: read oid 37 snap -1 2024-09-21T09:58:56.507 INFO:tasks.rados.rados.0.smithi033.stdout:2420: expect (ObjNum 731 snap 210 seq_num 731) 2024-09-21T09:58:56.507 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 222 2024-09-21T09:58:56.507 INFO:tasks.rados.rados.0.smithi033.stdout:2421: seq_num 774 ranges {3727360=565248,4292608=401408} 2024-09-21T09:58:56.532 INFO:tasks.rados.rados.0.smithi033.stdout:2421: writing smithi03385152-43 from 3727360 to 4292608 tid 1 2024-09-21T09:58:56.534 INFO:tasks.rados.rados.0.smithi033.stdout:2421: writing smithi03385152-43 from 4292608 to 4694016 tid 2 2024-09-21T09:58:56.534 INFO:tasks.rados.rados.0.smithi033.stdout:2422: read oid 32 snap -1 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2422: expect (ObjNum 752 snap 219 seq_num 752) 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2423: snap_create 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2411: finishing rollback tid 0 to smithi03385152-15 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2411: finishing rollback tid 1 to smithi03385152-15 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2411: finishing rollback tid 2 to smithi03385152-15 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 925 (ObjNum 684 snap 188 seq_num 684) dirty exists 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2409: finishing rollback tid 0 to smithi03385152-27 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2409: finishing rollback tid 1 to smithi03385152-27 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2409: finishing rollback tid 2 to smithi03385152-27 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1592 (ObjNum 697 snap 193 seq_num 697) dirty exists 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2412: finishing rollback tid 0 to smithi03385152-21 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2413: finishing rollback tid 0 to smithi03385152-42 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2412: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:2412: finishing rollback tid 2 to smithi03385152-21 2024-09-21T09:58:56.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1484 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T09:58:56.536 INFO:tasks.rados.rados.0.smithi033.stdout:2413: finishing rollback tid 1 to smithi03385152-42 2024-09-21T09:58:56.536 INFO:tasks.rados.rados.0.smithi033.stdout:2413: finishing rollback tid 2 to smithi03385152-42 2024-09-21T09:58:56.536 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 850 (ObjNum 613 snap 163 seq_num 613) dirty exists 2024-09-21T09:58:56.536 INFO:tasks.rados.rados.0.smithi033.stdout:2416: expect (ObjNum 765 snap 219 seq_num 765) 2024-09-21T09:58:56.560 INFO:tasks.rados.rados.0.smithi033.stdout:2417: finishing write tid 1 to smithi03385152-41 2024-09-21T09:58:56.561 INFO:tasks.rados.rados.0.smithi033.stdout:2417: finishing write tid 2 to smithi03385152-41 2024-09-21T09:58:56.561 INFO:tasks.rados.rados.0.smithi033.stdout:2417: finishing write tid 3 to smithi03385152-41 2024-09-21T09:58:56.561 INFO:tasks.rados.rados.0.smithi033.stdout:2417: finishing write tid 4 to smithi03385152-41 2024-09-21T09:58:56.561 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 854 (ObjNum 773 snap 222 seq_num 773) dirty exists 2024-09-21T09:58:56.561 INFO:tasks.rados.rados.0.smithi033.stdout:2417: left oid 41 (ObjNum 773 snap 222 seq_num 773) 2024-09-21T09:58:56.561 INFO:tasks.rados.rados.0.smithi033.stdout:2418: expect (ObjNum 691 snap 192 seq_num 691) 2024-09-21T09:58:56.615 INFO:tasks.rados.rados.0.smithi033.stdout:2410: expect (ObjNum 663 snap 184 seq_num 663) 2024-09-21T09:58:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:56 smithi105 ceph-mon[86649]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T09:58:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:56 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:56.990 INFO:tasks.rados.rados.0.smithi033.stdout:2420: expect (ObjNum 731 snap 210 seq_num 731) 2024-09-21T09:58:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:56 smithi033 ceph-mon[98546]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T09:58:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:56 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:56.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:56 smithi033 ceph-mon[96568]: osdmap e506: 8 total, 8 up, 8 in 2024-09-21T09:58:56.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:56 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:56.996 INFO:tasks.rados.rados.0.smithi033.stdout:2421: finishing write tid 1 to smithi03385152-43 2024-09-21T09:58:56.996 INFO:tasks.rados.rados.0.smithi033.stdout:2421: finishing write tid 2 to smithi03385152-43 2024-09-21T09:58:56.997 INFO:tasks.rados.rados.0.smithi033.stdout:2421: finishing write tid 3 to smithi03385152-43 2024-09-21T09:58:56.997 INFO:tasks.rados.rados.0.smithi033.stdout:2421: finishing write tid 4 to smithi03385152-43 2024-09-21T09:58:56.997 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1039 (ObjNum 774 snap 222 seq_num 774) dirty exists 2024-09-21T09:58:56.997 INFO:tasks.rados.rados.0.smithi033.stdout:2421: left oid 43 (ObjNum 774 snap 222 seq_num 774) 2024-09-21T09:58:56.997 INFO:tasks.rados.rados.0.smithi033.stdout:2422: expect (ObjNum 752 snap 219 seq_num 752) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2409: done (12 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2410: done (11 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2411: done (10 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2412: done (9 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2413: done (8 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2414: done (7 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2416: done (6 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2417: done (5 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2418: done (4 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2420: done (3 left) 2024-09-21T09:58:57.519 INFO:tasks.rados.rados.0.smithi033.stdout:2421: done (2 left) 2024-09-21T09:58:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:2422: done (1 left) 2024-09-21T09:58:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:2423: done (0 left) 2024-09-21T09:58:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:2424: rmattr oid 38 current snap is 223 2024-09-21T09:58:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:2425: read oid 42 snap 210 2024-09-21T09:58:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:2425: expect (ObjNum 683 snap 187 seq_num 683) 2024-09-21T09:58:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:2426: read oid 18 snap 219 2024-09-21T09:58:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:2426: expect (ObjNum 767 snap 219 seq_num 767) 2024-09-21T09:58:57.520 INFO:tasks.rados.rados.0.smithi033.stdout:2427: delete oid 14 current snap is 223 2024-09-21T09:58:57.523 INFO:tasks.rados.rados.0.smithi033.stdout:2427: done (3 left) 2024-09-21T09:58:57.523 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 223 2024-09-21T09:58:57.523 INFO:tasks.rados.rados.0.smithi033.stdout:2428: seq_num 776 ranges {4694016=606208,5300224=401408} 2024-09-21T09:58:57.552 INFO:tasks.rados.rados.0.smithi033.stdout:2428: writing smithi03385152-43 from 4694016 to 5300224 tid 1 2024-09-21T09:58:57.555 INFO:tasks.rados.rados.0.smithi033.stdout:2428: writing smithi03385152-43 from 5300224 to 5701632 tid 2 2024-09-21T09:58:57.555 INFO:tasks.rados.rados.0.smithi033.stdout:2429: delete oid 32 current snap is 223 2024-09-21T09:58:57.555 INFO:tasks.rados.rados.0.smithi033.stdout:2426: expect (ObjNum 767 snap 219 seq_num 767) 2024-09-21T09:58:57.578 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 860 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:58:57.578 INFO:tasks.rados.rados.0.smithi033.stdout:2425: expect (ObjNum 683 snap 187 seq_num 683) 2024-09-21T09:58:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:57 smithi105 ceph-mon[86649]: pgmap v390: 121 pgs: 121 active+clean; 278 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-21T09:58:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:57 smithi105 ceph-mon[86649]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T09:58:57.804 INFO:tasks.rados.rados.0.smithi033.stdout:2428: finishing write tid 1 to smithi03385152-43 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2428: finishing write tid 2 to smithi03385152-43 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2428: finishing write tid 3 to smithi03385152-43 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2428: finishing write tid 4 to smithi03385152-43 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1042 (ObjNum 776 snap 223 seq_num 776) dirty exists 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2428: left oid 43 (ObjNum 776 snap 223 seq_num 776) 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2424: done (4 left) 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2425: done (3 left) 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2426: done (2 left) 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2428: done (1 left) 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2429: done (0 left) 2024-09-21T09:58:57.805 INFO:tasks.rados.rados.0.smithi033.stdout:2430: snap_create 2024-09-21T09:58:57.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:57 smithi033 ceph-mon[98546]: pgmap v390: 121 pgs: 121 active+clean; 278 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-21T09:58:57.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:57 smithi033 ceph-mon[98546]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T09:58:57.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:57 smithi033 ceph-mon[96568]: pgmap v390: 121 pgs: 121 active+clean; 278 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-21T09:58:57.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:57 smithi033 ceph-mon[96568]: osdmap e507: 8 total, 8 up, 8 in 2024-09-21T09:58:58.493 INFO:tasks.rados.rados.0.smithi033.stdout:2430: done (0 left) 2024-09-21T09:58:58.493 INFO:tasks.rados.rados.0.smithi033.stdout:2431: snap_remove snap 209 2024-09-21T09:58:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:58 smithi105 ceph-mon[86649]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T09:58:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:58 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:58:58.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:58 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:58:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:58 smithi105 ceph-mon[86649]: Upgrade: osd.6 is safe to restart 2024-09-21T09:58:58.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:58 smithi105 ceph-mon[86649]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T09:58:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[98546]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T09:58:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:58:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:58:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[98546]: Upgrade: osd.6 is safe to restart 2024-09-21T09:58:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[98546]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T09:58:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[96568]: osdmap e508: 8 total, 8 up, 8 in 2024-09-21T09:58:58.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16} : dispatch 2024-09-21T09:58:58.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-09-21T09:58:58.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[96568]: Upgrade: osd.6 is safe to restart 2024-09-21T09:58:58.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:58 smithi033 ceph-mon[96568]: osdmap e509: 8 total, 8 up, 8 in 2024-09-21T09:58:59.497 INFO:tasks.rados.rados.0.smithi033.stdout:2431: done (0 left) 2024-09-21T09:58:59.497 INFO:tasks.rados.rados.0.smithi033.stdout:2432: copy_from oid 33 from oid 14 current snap is 224 2024-09-21T09:58:59.497 INFO:tasks.rados.rados.0.smithi033.stdout:2433: rmattr oid 20 current snap is 224 2024-09-21T09:58:59.498 INFO:tasks.rados.rados.0.smithi033.stdout:2434: snap_remove snap 218 2024-09-21T09:58:59.771 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:59 smithi105 ceph-mon[86649]: pgmap v393: 121 pgs: 121 active+clean; 278 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 29 op/s 2024-09-21T09:58:59.771 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:59 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:59.771 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:59 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:58:59.771 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:59 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:59.772 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:58:59 smithi105 ceph-mon[86649]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T09:58:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[98546]: pgmap v393: 121 pgs: 121 active+clean; 278 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 29 op/s 2024-09-21T09:58:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:58:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[98546]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T09:58:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[96568]: pgmap v393: 121 pgs: 121 active+clean; 278 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 2.0 MiB/s wr, 29 op/s 2024-09-21T09:58:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:58:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-21T09:58:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:58:59.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:58:59 smithi033 ceph-mon[96568]: osdmap e510: 8 total, 8 up, 8 in 2024-09-21T09:59:00.501 INFO:tasks.rados.rados.0.smithi033.stdout:2434: done (2 left) 2024-09-21T09:59:00.501 INFO:tasks.rados.rados.0.smithi033.stdout:2435: snap_create 2024-09-21T09:59:00.501 INFO:tasks.rados.rados.0.smithi033.stdout:2432: finishing copy_from to smithi03385152-33 2024-09-21T09:59:00.501 INFO:tasks.rados.rados.0.smithi033.stdout:2432: got expected ENOENT (src dne) 2024-09-21T09:59:00.501 INFO:tasks.rados.rados.0.smithi033.stdout:2432: finishing copy_from racing read to smithi03385152-33 2024-09-21T09:59:00.501 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1485 (ObjNum 757 snap 219 seq_num 757) dirty exists 2024-09-21T09:59:00.525 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:59:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:59:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:00 smithi105 ceph-mon[86649]: Upgrade: Updating osd.6 2024-09-21T09:59:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:00 smithi105 ceph-mon[86649]: Deploying daemon osd.6 on smithi105 2024-09-21T09:59:00.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:00 smithi105 ceph-mon[86649]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T09:59:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:00 smithi033 ceph-mon[96568]: Upgrade: Updating osd.6 2024-09-21T09:59:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:00 smithi033 ceph-mon[96568]: Deploying daemon osd.6 on smithi105 2024-09-21T09:59:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:00 smithi033 ceph-mon[96568]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T09:59:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:00 smithi033 ceph-mon[98546]: Upgrade: Updating osd.6 2024-09-21T09:59:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:00 smithi033 ceph-mon[98546]: Deploying daemon osd.6 on smithi105 2024-09-21T09:59:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:00 smithi033 ceph-mon[98546]: osdmap e511: 8 total, 8 up, 8 in 2024-09-21T09:59:01.500 INFO:tasks.rados.rados.0.smithi033.stdout:2432: done (2 left) 2024-09-21T09:59:01.501 INFO:tasks.rados.rados.0.smithi033.stdout:2433: done (1 left) 2024-09-21T09:59:01.501 INFO:tasks.rados.rados.0.smithi033.stdout:2435: done (0 left) 2024-09-21T09:59:01.501 INFO:tasks.rados.rados.0.smithi033.stdout:2436: read oid 47 snap -1 2024-09-21T09:59:01.501 INFO:tasks.rados.rados.0.smithi033.stdout:2436: expect deleted 2024-09-21T09:59:01.501 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 225 2024-09-21T09:59:01.501 INFO:tasks.rados.rados.0.smithi033.stdout:2437: seq_num 778 ranges {3244032=638976,3883008=401408} 2024-09-21T09:59:01.521 INFO:tasks.rados.rados.0.smithi033.stdout:2437: writing smithi03385152-12 from 3244032 to 3883008 tid 1 2024-09-21T09:59:01.524 INFO:tasks.rados.rados.0.smithi033.stdout:2437: writing smithi03385152-12 from 3883008 to 4284416 tid 2 2024-09-21T09:59:01.524 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 225 2024-09-21T09:59:01.524 INFO:tasks.rados.rados.0.smithi033.stdout:2438: seq_num 779 ranges {2220032=655360,2875392=401408} 2024-09-21T09:59:01.539 INFO:tasks.rados.rados.0.smithi033.stdout:2438: writing smithi03385152-8 from 2220032 to 2875392 tid 1 2024-09-21T09:59:01.542 INFO:tasks.rados.rados.0.smithi033.stdout:2438: writing smithi03385152-8 from 2875392 to 3276800 tid 2 2024-09-21T09:59:01.542 INFO:tasks.rados.rados.0.smithi033.stdout:2439: delete oid 31 current snap is 225 2024-09-21T09:59:01.542 INFO:tasks.rados.rados.0.smithi033.stdout:2437: finishing write tid 1 to smithi03385152-12 2024-09-21T09:59:01.542 INFO:tasks.rados.rados.0.smithi033.stdout:2437: finishing write tid 2 to smithi03385152-12 2024-09-21T09:59:01.542 INFO:tasks.rados.rados.0.smithi033.stdout:2437: finishing write tid 3 to smithi03385152-12 2024-09-21T09:59:01.543 INFO:tasks.rados.rados.0.smithi033.stdout:2437: finishing write tid 4 to smithi03385152-12 2024-09-21T09:59:01.543 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1480 (ObjNum 778 snap 225 seq_num 778) dirty exists 2024-09-21T09:59:01.543 INFO:tasks.rados.rados.0.smithi033.stdout:2437: left oid 12 (ObjNum 778 snap 225 seq_num 778) 2024-09-21T09:59:01.545 INFO:tasks.rados.rados.0.smithi033.stdout:2438: finishing write tid 1 to smithi03385152-8 2024-09-21T09:59:01.545 INFO:tasks.rados.rados.0.smithi033.stdout:2438: finishing write tid 2 to smithi03385152-8 2024-09-21T09:59:01.545 INFO:tasks.rados.rados.0.smithi033.stdout:2438: finishing write tid 3 to smithi03385152-8 2024-09-21T09:59:01.547 INFO:tasks.rados.rados.0.smithi033.stdout:2438: finishing write tid 4 to smithi03385152-8 2024-09-21T09:59:01.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 928 (ObjNum 779 snap 225 seq_num 779) dirty exists 2024-09-21T09:59:01.547 INFO:tasks.rados.rados.0.smithi033.stdout:2438: left oid 8 (ObjNum 779 snap 225 seq_num 779) 2024-09-21T09:59:01.548 INFO:tasks.rados.rados.0.smithi033.stdout:2436: done (3 left) 2024-09-21T09:59:01.548 INFO:tasks.rados.rados.0.smithi033.stdout:2437: done (2 left) 2024-09-21T09:59:01.548 INFO:tasks.rados.rados.0.smithi033.stdout:2438: done (1 left) 2024-09-21T09:59:01.548 INFO:tasks.rados.rados.0.smithi033.stdout:2439: done (0 left) 2024-09-21T09:59:01.548 INFO:tasks.rados.rados.0.smithi033.stdout:2440: delete oid 43 current snap is 225 2024-09-21T09:59:01.551 INFO:tasks.rados.rados.0.smithi033.stdout:2440: done (0 left) 2024-09-21T09:59:01.551 INFO:tasks.rados.rados.0.smithi033.stdout:2441: read oid 13 snap -1 2024-09-21T09:59:01.551 INFO:tasks.rados.rados.0.smithi033.stdout:2441: expect (ObjNum 665 snap 185 seq_num 665) 2024-09-21T09:59:01.551 INFO:tasks.rados.rados.0.smithi033.stdout:2442: snap_create 2024-09-21T09:59:01.570 INFO:tasks.rados.rados.0.smithi033.stdout:2441: expect (ObjNum 665 snap 185 seq_num 665) 2024-09-21T09:59:01.883 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:01 smithi105 ceph-mon[86649]: pgmap v396: 121 pgs: 121 active+clean; 280 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 714 KiB/s wr, 18 op/s 2024-09-21T09:59:01.883 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:01.883 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:01.883 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:01 smithi105 ceph-mon[86649]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:59:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:59:01] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.51.0" 2024-09-21T09:59:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-mon[98546]: pgmap v396: 121 pgs: 121 active+clean; 280 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 714 KiB/s wr, 18 op/s 2024-09-21T09:59:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-mon[98546]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:59:01.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-mon[96568]: pgmap v396: 121 pgs: 121 active+clean; 280 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 714 KiB/s wr, 18 op/s 2024-09-21T09:59:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:01 smithi033 ceph-mon[96568]: osdmap e512: 8 total, 8 up, 8 in 2024-09-21T09:59:02.481 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:02 smithi105 systemd[1]: Stopping Ceph osd.6 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:59:02.510 INFO:tasks.rados.rados.0.smithi033.stdout:2441: done (1 left) 2024-09-21T09:59:02.510 INFO:tasks.rados.rados.0.smithi033.stdout:2442: done (0 left) 2024-09-21T09:59:02.510 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 226 2024-09-21T09:59:02.510 INFO:tasks.rados.rados.0.smithi033.stdout:2443: seq_num 782 ranges {835584=704512,1540096=401408} 2024-09-21T09:59:02.518 INFO:tasks.rados.rados.0.smithi033.stdout:2443: writing smithi03385152-33 from 835584 to 1540096 tid 1 2024-09-21T09:59:02.520 INFO:tasks.rados.rados.0.smithi033.stdout:2443: writing smithi03385152-33 from 1540096 to 1941504 tid 2 2024-09-21T09:59:02.520 INFO:tasks.rados.rados.0.smithi033.stdout:2444: snap_create 2024-09-21T09:59:02.562 INFO:tasks.rados.rados.0.smithi033.stdout:2443: finishing write tid 1 to smithi03385152-33 2024-09-21T09:59:02.562 INFO:tasks.rados.rados.0.smithi033.stdout:2443: finishing write tid 2 to smithi03385152-33 2024-09-21T09:59:02.562 INFO:tasks.rados.rados.0.smithi033.stdout:2443: finishing write tid 3 to smithi03385152-33 2024-09-21T09:59:02.563 INFO:tasks.rados.rados.0.smithi033.stdout:2443: finishing write tid 4 to smithi03385152-33 2024-09-21T09:59:02.563 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1595 (ObjNum 782 snap 226 seq_num 782) dirty exists 2024-09-21T09:59:02.563 INFO:tasks.rados.rados.0.smithi033.stdout:2443: left oid 33 (ObjNum 782 snap 226 seq_num 782) 2024-09-21T09:59:02.772 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:02 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[49024]: 2024-09-21T09:59:02.478+0000 7f09430de640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:59:02.772 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:02 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[49024]: 2024-09-21T09:59:02.478+0000 7f09430de640 -1 osd.6 512 *** Got signal Terminated *** 2024-09-21T09:59:02.772 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:02 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[49024]: 2024-09-21T09:59:02.478+0000 7f09430de640 -1 osd.6 512 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:59:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:02 smithi105 ceph-mon[86649]: osd.6 marked itself down and dead 2024-09-21T09:59:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:02 smithi105 ceph-mon[86649]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:59:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:02 smithi105 ceph-mon[86649]: osdmap e513: 8 total, 7 up, 8 in 2024-09-21T09:59:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:02 smithi033 ceph-mon[98546]: osd.6 marked itself down and dead 2024-09-21T09:59:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:02 smithi033 ceph-mon[98546]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:59:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:02 smithi033 ceph-mon[98546]: osdmap e513: 8 total, 7 up, 8 in 2024-09-21T09:59:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:02 smithi033 ceph-mon[96568]: osd.6 marked itself down and dead 2024-09-21T09:59:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:02 smithi033 ceph-mon[96568]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:59:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:02 smithi033 ceph-mon[96568]: osdmap e513: 8 total, 7 up, 8 in 2024-09-21T09:59:03.348 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:03 smithi105 podman[101239]: 2024-09-21 09:59:03.051627529 +0000 UTC m=+0.685975954 container died 37fd995b47ba8fd6394c406fb818a32dd02b22f236a18122405565fa9614e7e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T09:59:03.498 INFO:tasks.rados.rados.0.smithi033.stdout:2443: done (1 left) 2024-09-21T09:59:03.498 INFO:tasks.rados.rados.0.smithi033.stdout:2444: done (0 left) 2024-09-21T09:59:03.498 INFO:tasks.rados.rados.0.smithi033.stdout:2445: delete oid 46 current snap is 227 2024-09-21T09:59:03.517 INFO:tasks.rados.rados.0.smithi033.stdout:2445: done (0 left) 2024-09-21T09:59:03.517 INFO:tasks.rados.rados.0.smithi033.stdout:2446: delete oid 40 current snap is 227 2024-09-21T09:59:03.520 INFO:tasks.rados.rados.0.smithi033.stdout:2446: done (0 left) 2024-09-21T09:59:03.520 INFO:tasks.rados.rados.0.smithi033.stdout:2447: snap_create 2024-09-21T09:59:03.923 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:03 smithi105 ceph-mon[86649]: pgmap v399: 121 pgs: 121 active+clean; 280 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 714 KiB/s wr, 17 op/s 2024-09-21T09:59:03.923 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:03 smithi105 ceph-mon[86649]: osdmap e514: 8 total, 7 up, 8 in 2024-09-21T09:59:03.923 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:03 smithi105 podman[101239]: 2024-09-21 09:59:03.620170739 +0000 UTC m=+1.254519162 container cleanup 37fd995b47ba8fd6394c406fb818a32dd02b22f236a18122405565fa9614e7e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, ceph=True, GIT_CLEAN=True) 2024-09-21T09:59:03.923 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:03 smithi105 bash[101239]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6 2024-09-21T09:59:04.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:03 smithi033 ceph-mon[98546]: pgmap v399: 121 pgs: 121 active+clean; 280 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 714 KiB/s wr, 17 op/s 2024-09-21T09:59:04.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:03 smithi033 ceph-mon[98546]: osdmap e514: 8 total, 7 up, 8 in 2024-09-21T09:59:04.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:03 smithi033 ceph-mon[96568]: pgmap v399: 121 pgs: 121 active+clean; 280 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 714 KiB/s wr, 17 op/s 2024-09-21T09:59:04.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:03 smithi033 ceph-mon[96568]: osdmap e514: 8 total, 7 up, 8 in 2024-09-21T09:59:04.273 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:03 smithi105 podman[101259]: 2024-09-21 09:59:03.921706539 +0000 UTC m=+0.870865405 container remove 37fd995b47ba8fd6394c406fb818a32dd02b22f236a18122405565fa9614e7e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=quincy-fbd1a6a, io.buildah.version=1.37.2) 2024-09-21T09:59:04.498 INFO:tasks.rados.rados.0.smithi033.stdout:2447: done (0 left) 2024-09-21T09:59:04.499 INFO:tasks.rados.rados.0.smithi033.stdout:2448: read oid 50 snap -1 2024-09-21T09:59:04.499 INFO:tasks.rados.rados.0.smithi033.stdout:2448: expect deleted 2024-09-21T09:59:04.499 INFO:tasks.rados.rados.0.smithi033.stdout:2449: read oid 18 snap -1 2024-09-21T09:59:04.499 INFO:tasks.rados.rados.0.smithi033.stdout:2449: expect (ObjNum 767 snap 219 seq_num 767) 2024-09-21T09:59:04.499 INFO:tasks.rados.rados.0.smithi033.stdout:2450: rmattr oid 13 current snap is 228 2024-09-21T09:59:04.499 INFO:tasks.rados.rados.0.smithi033.stdout:2451: rmattr oid 5 current snap is 228 2024-09-21T09:59:04.499 INFO:tasks.rados.rados.0.smithi033.stdout:2452: snap_create 2024-09-21T09:59:04.505 INFO:tasks.rados.rados.0.smithi033.stdout:2449: expect (ObjNum 767 snap 219 seq_num 767) 2024-09-21T09:59:04.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 1012 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:59:04.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 1614 (ObjNum 665 snap 185 seq_num 665) dirty exists 2024-09-21T09:59:04.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:04 smithi105 podman[101356]: 2024-09-21 09:59:04.272648767 +0000 UTC m=+0.104409934 container create fd5233c6803a71c16b9df805cf71758e08ed7de5813a26e4630feffb56fa5b76 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True) 2024-09-21T09:59:04.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:04 smithi105 podman[101356]: 2024-09-21 09:59:04.184247053 +0000 UTC m=+0.016008232 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:59:04.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:04 smithi105 podman[101356]: 2024-09-21 09:59:04.483349299 +0000 UTC m=+0.315110473 container init fd5233c6803a71c16b9df805cf71758e08ed7de5813a26e4630feffb56fa5b76 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.schema-version=1.0) 2024-09-21T09:59:04.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:04 smithi105 podman[101356]: 2024-09-21 09:59:04.488077557 +0000 UTC m=+0.319838723 container start fd5233c6803a71c16b9df805cf71758e08ed7de5813a26e4630feffb56fa5b76 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:59:05.030 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:04 smithi105 podman[101356]: 2024-09-21 09:59:04.591682884 +0000 UTC m=+0.423444054 container attach fd5233c6803a71c16b9df805cf71758e08ed7de5813a26e4630feffb56fa5b76 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, RELEASE=main-62da54b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-21T09:59:05.030 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:04 smithi105 podman[101356]: 2024-09-21 09:59:04.68613394 +0000 UTC m=+0.517895118 container died fd5233c6803a71c16b9df805cf71758e08ed7de5813a26e4630feffb56fa5b76 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.37.2) 2024-09-21T09:59:05.500 INFO:tasks.rados.rados.0.smithi033.stdout:2448: done (4 left) 2024-09-21T09:59:05.500 INFO:tasks.rados.rados.0.smithi033.stdout:2449: done (3 left) 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:2450: done (2 left) 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:2451: done (1 left) 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:2452: done (0 left) 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:2453: copy_from oid 48 from oid 32 current snap is 229 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:2454: rollback oid 19 current snap is 229 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 19 to 215 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:2455: copy_from oid 36 from oid 37 current snap is 229 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 229 2024-09-21T09:59:05.501 INFO:tasks.rados.rados.0.smithi033.stdout:2456: seq_num 785 ranges {0=753664,753664=401408} 2024-09-21T09:59:05.505 INFO:tasks.rados.rados.0.smithi033.stdout:2456: writing smithi03385152-39 from 0 to 753664 tid 1 2024-09-21T09:59:05.509 INFO:tasks.rados.rados.0.smithi033.stdout:2456: writing smithi03385152-39 from 753664 to 1155072 tid 2 2024-09-21T09:59:05.509 INFO:tasks.rados.rados.0.smithi033.stdout:2457: snap_create 2024-09-21T09:59:05.509 INFO:tasks.rados.rados.0.smithi033.stdout:2453: finishing copy_from to smithi03385152-48 2024-09-21T09:59:05.509 INFO:tasks.rados.rados.0.smithi033.stdout:2453: got expected ENOENT (src dne) 2024-09-21T09:59:05.509 INFO:tasks.rados.rados.0.smithi033.stdout:2453: finishing copy_from racing read to smithi03385152-48 2024-09-21T09:59:05.509 INFO:tasks.rados.rados.0.smithi033.stdout:2454: finishing rollback tid 1 to smithi03385152-19 2024-09-21T09:59:05.510 INFO:tasks.rados.rados.0.smithi033.stdout:2454: finishing rollback tid 2 to smithi03385152-19 2024-09-21T09:59:05.510 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1034 (ObjNum 741 snap 215 seq_num 741) dirty exists 2024-09-21T09:59:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:2456: finishing write tid 1 to smithi03385152-39 2024-09-21T09:59:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:2456: finishing write tid 2 to smithi03385152-39 2024-09-21T09:59:05.512 INFO:tasks.rados.rados.0.smithi033.stdout:2456: finishing write tid 3 to smithi03385152-39 2024-09-21T09:59:05.513 INFO:tasks.rados.rados.0.smithi033.stdout:2455: finishing copy_from racing read to smithi03385152-36 2024-09-21T09:59:05.514 INFO:tasks.rados.rados.0.smithi033.stdout:2456: finishing write tid 4 to smithi03385152-39 2024-09-21T09:59:05.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1015 (ObjNum 785 snap 229 seq_num 785) dirty exists 2024-09-21T09:59:05.514 INFO:tasks.rados.rados.0.smithi033.stdout:2456: left oid 39 (ObjNum 785 snap 229 seq_num 785) 2024-09-21T09:59:05.514 INFO:tasks.rados.rados.0.smithi033.stdout:2455: finishing copy_from to smithi03385152-36 2024-09-21T09:59:05.514 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1034 (ObjNum 731 snap 210 seq_num 731) dirty exists 2024-09-21T09:59:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:05 smithi105 ceph-mon[86649]: pgmap v402: 121 pgs: 3 active+undersized, 7 peering, 13 stale+active+clean, 98 active+clean; 280 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:59:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:05 smithi105 ceph-mon[86649]: osdmap e515: 8 total, 7 up, 8 in 2024-09-21T09:59:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:05 smithi105 ceph-mon[86649]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:59:05.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:05 smithi105 podman[101399]: 2024-09-21 09:59:05.395926706 +0000 UTC m=+0.704376052 container remove fd5233c6803a71c16b9df805cf71758e08ed7de5813a26e4630feffb56fa5b76 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T09:59:05.781 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:05 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.6.service: Deactivated successfully. 2024-09-21T09:59:05.781 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:05 smithi105 systemd[1]: Stopped Ceph osd.6 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:59:05.781 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:05 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.6.service: Consumed 13.257s CPU time. 2024-09-21T09:59:05.781 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:05 smithi105 systemd[1]: Starting Ceph osd.6 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:59:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:05 smithi033 ceph-mon[98546]: pgmap v402: 121 pgs: 3 active+undersized, 7 peering, 13 stale+active+clean, 98 active+clean; 280 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:59:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:05 smithi033 ceph-mon[98546]: osdmap e515: 8 total, 7 up, 8 in 2024-09-21T09:59:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:05 smithi033 ceph-mon[98546]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:59:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:05 smithi033 ceph-mon[96568]: pgmap v402: 121 pgs: 3 active+undersized, 7 peering, 13 stale+active+clean, 98 active+clean; 280 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T09:59:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:05 smithi033 ceph-mon[96568]: osdmap e515: 8 total, 7 up, 8 in 2024-09-21T09:59:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:05 smithi033 ceph-mon[96568]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:59:06.109 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:05 smithi105 podman[101502]: 2024-09-21 09:59:05.740585163 +0000 UTC m=+0.015886609 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:59:06.109 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:05 smithi105 podman[101502]: 2024-09-21 09:59:05.855263752 +0000 UTC m=+0.130565226 container create 938ebace65f2b2f4fb84e5676c49be8153d37d643847c150de21bf593fe44141 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:59:06.109 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 podman[101502]: 2024-09-21 09:59:06.107773718 +0000 UTC m=+0.383075153 container init 938ebace65f2b2f4fb84e5676c49be8153d37d643847c150de21bf593fe44141 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-21T09:59:06.512 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 podman[101502]: 2024-09-21 09:59:06.113193867 +0000 UTC m=+0.388495299 container start 938ebace65f2b2f4fb84e5676c49be8153d37d643847c150de21bf593fe44141 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T09:59:06.512 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 podman[101502]: 2024-09-21 09:59:06.199514683 +0000 UTC m=+0.474816132 container attach 938ebace65f2b2f4fb84e5676c49be8153d37d643847c150de21bf593fe44141 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate, GIT_BRANCH=HEAD, ceph=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T09:59:06.512 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:59:06.512 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 bash[101502]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:59:06.512 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:59:06.512 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 bash[101502]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:59:06.518 INFO:tasks.rados.rados.0.smithi033.stdout:2453: done (4 left) 2024-09-21T09:59:06.518 INFO:tasks.rados.rados.0.smithi033.stdout:2454: done (3 left) 2024-09-21T09:59:06.518 INFO:tasks.rados.rados.0.smithi033.stdout:2455: done (2 left) 2024-09-21T09:59:06.518 INFO:tasks.rados.rados.0.smithi033.stdout:2456: done (1 left) 2024-09-21T09:59:06.518 INFO:tasks.rados.rados.0.smithi033.stdout:2457: done (0 left) 2024-09-21T09:59:06.519 INFO:tasks.rados.rados.0.smithi033.stdout:2458: read oid 28 snap -1 2024-09-21T09:59:06.519 INFO:tasks.rados.rados.0.smithi033.stdout:2458: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:59:06.519 INFO:tasks.rados.rados.0.smithi033.stdout:2459: read oid 50 snap -1 2024-09-21T09:59:06.519 INFO:tasks.rados.rados.0.smithi033.stdout:2459: expect deleted 2024-09-21T09:59:06.519 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 230 2024-09-21T09:59:06.519 INFO:tasks.rados.rados.0.smithi033.stdout:2460: seq_num 786 ranges {2826240=770048,3596288=401408} 2024-09-21T09:59:06.537 INFO:tasks.rados.rados.0.smithi033.stdout:2460: writing smithi03385152-15 from 2826240 to 3596288 tid 1 2024-09-21T09:59:06.540 INFO:tasks.rados.rados.0.smithi033.stdout:2460: writing smithi03385152-15 from 3596288 to 3997696 tid 2 2024-09-21T09:59:06.540 INFO:tasks.rados.rados.0.smithi033.stdout:2461: snap_remove snap 226 2024-09-21T09:59:06.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:06 smithi105 ceph-mon[86649]: osdmap e516: 8 total, 7 up, 8 in 2024-09-21T09:59:06.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T09:59:06.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 bash[101502]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T09:59:06.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:59:06.781 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:06 smithi105 bash[101502]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-21T09:59:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:06 smithi033 ceph-mon[96568]: osdmap e516: 8 total, 7 up, 8 in 2024-09-21T09:59:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:06 smithi033 ceph-mon[98546]: osdmap e516: 8 total, 7 up, 8 in 2024-09-21T09:59:07.280 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:59:07.280 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 bash[101502]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-21T09:59:07.280 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:59:07.281 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 bash[101502]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-21T09:59:07.281 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T09:59:07.281 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 bash[101502]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-09-21T09:59:07.281 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T09:59:07.281 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 bash[101502]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-09-21T09:59:07.281 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate[101517]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-21T09:59:07.281 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 bash[101502]: --> ceph-volume raw activate successful for osd ID: 6 2024-09-21T09:59:07.281 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 podman[101502]: 2024-09-21 09:59:07.031247319 +0000 UTC m=+1.306548755 container died 938ebace65f2b2f4fb84e5676c49be8153d37d643847c150de21bf593fe44141 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-21T09:59:07.521 INFO:tasks.rados.rados.0.smithi033.stdout:2461: done (3 left) 2024-09-21T09:59:07.521 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 230 2024-09-21T09:59:07.521 INFO:tasks.rados.rados.0.smithi033.stdout:2462: seq_num 787 ranges {3162112=786432,3948544=401408} 2024-09-21T09:59:07.542 INFO:tasks.rados.rados.0.smithi033.stdout:2462: writing smithi03385152-25 from 3162112 to 3948544 tid 1 2024-09-21T09:59:07.544 INFO:tasks.rados.rados.0.smithi033.stdout:2462: writing smithi03385152-25 from 3948544 to 4349952 tid 2 2024-09-21T09:59:07.545 INFO:tasks.rados.rados.0.smithi033.stdout:2463: rollback oid 23 current snap is 230 2024-09-21T09:59:07.545 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 23 to 173 2024-09-21T09:59:07.545 INFO:tasks.rados.rados.0.smithi033.stdout:2458: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:59:07.610 INFO:tasks.rados.rados.0.smithi033.stdout:2460: finishing write tid 1 to smithi03385152-15 2024-09-21T09:59:07.610 INFO:tasks.rados.rados.0.smithi033.stdout:2460: finishing write tid 2 to smithi03385152-15 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2460: finishing write tid 3 to smithi03385152-15 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2460: finishing write tid 4 to smithi03385152-15 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 931 (ObjNum 786 snap 230 seq_num 786) dirty exists 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2460: left oid 15 (ObjNum 786 snap 230 seq_num 786) 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2462: finishing write tid 1 to smithi03385152-25 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2462: finishing write tid 2 to smithi03385152-25 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2462: finishing write tid 3 to smithi03385152-25 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2458: done (4 left) 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2459: done (3 left) 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2460: done (2 left) 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2464: read oid 26 snap 220 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2464: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:07.611 INFO:tasks.rados.rados.0.smithi033.stdout:2465: read oid 13 snap 227 2024-09-21T09:59:07.612 INFO:tasks.rados.rados.0.smithi033.stdout:2465: expect (ObjNum 665 snap 185 seq_num 665) 2024-09-21T09:59:07.612 INFO:tasks.rados.rados.0.smithi033.stdout:2466: delete oid 1 current snap is 230 2024-09-21T09:59:07.612 INFO:tasks.rados.rados.0.smithi033.stdout:2463: finishing rollback tid 1 to smithi03385152-23 2024-09-21T09:59:07.612 INFO:tasks.rados.rados.0.smithi033.stdout:2463: finishing rollback tid 2 to smithi03385152-23 2024-09-21T09:59:07.612 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1487 (ObjNum 627 snap 171 seq_num 627) dirty exists 2024-09-21T09:59:07.612 INFO:tasks.rados.rados.0.smithi033.stdout:2462: finishing write tid 4 to smithi03385152-25 2024-09-21T09:59:07.612 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 932 (ObjNum 787 snap 230 seq_num 787) dirty exists 2024-09-21T09:59:07.612 INFO:tasks.rados.rados.0.smithi033.stdout:2462: left oid 25 (ObjNum 787 snap 230 seq_num 787) 2024-09-21T09:59:07.615 INFO:tasks.rados.rados.0.smithi033.stdout:2462: done (4 left) 2024-09-21T09:59:07.615 INFO:tasks.rados.rados.0.smithi033.stdout:2463: done (3 left) 2024-09-21T09:59:07.615 INFO:tasks.rados.rados.0.smithi033.stdout:2466: done (2 left) 2024-09-21T09:59:07.615 INFO:tasks.rados.rados.0.smithi033.stdout:2467: snap_create 2024-09-21T09:59:07.617 INFO:tasks.rados.rados.0.smithi033.stdout:2464: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:07.631 INFO:tasks.rados.rados.0.smithi033.stdout:2465: expect (ObjNum 665 snap 185 seq_num 665) 2024-09-21T09:59:07.722 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:07 smithi105 ceph-mon[86649]: pgmap v405: 121 pgs: 20 active+undersized, 7 peering, 5 stale+active+clean, 8 active+undersized+degraded, 81 active+clean; 283 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 584 KiB/s wr, 6 op/s; 58/624 objects degraded (9.295%) 2024-09-21T09:59:07.722 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:07 smithi105 ceph-mon[86649]: Health check failed: Degraded data redundancy: 58/624 objects degraded (9.295%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:07.722 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:07 smithi105 ceph-mon[86649]: osdmap e517: 8 total, 7 up, 8 in 2024-09-21T09:59:07.973 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 podman[101679]: 2024-09-21 09:59:07.72072503 +0000 UTC m=+0.680146437 container remove 938ebace65f2b2f4fb84e5676c49be8153d37d643847c150de21bf593fe44141 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-activate, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731) 2024-09-21T09:59:07.973 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 podman[101744]: 2024-09-21 09:59:07.827497586 +0000 UTC m=+0.015662488 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:59:07.973 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:07 smithi105 podman[101744]: 2024-09-21 09:59:07.97185954 +0000 UTC m=+0.160024465 container create 132bc834f24d6afd074a39b8c8791c6833709b42389d6c9e312aaca69f98941a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-21T09:59:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:07 smithi033 ceph-mon[98546]: pgmap v405: 121 pgs: 20 active+undersized, 7 peering, 5 stale+active+clean, 8 active+undersized+degraded, 81 active+clean; 283 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 584 KiB/s wr, 6 op/s; 58/624 objects degraded (9.295%) 2024-09-21T09:59:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:07 smithi033 ceph-mon[98546]: Health check failed: Degraded data redundancy: 58/624 objects degraded (9.295%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:07 smithi033 ceph-mon[98546]: osdmap e517: 8 total, 7 up, 8 in 2024-09-21T09:59:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:07 smithi033 ceph-mon[96568]: pgmap v405: 121 pgs: 20 active+undersized, 7 peering, 5 stale+active+clean, 8 active+undersized+degraded, 81 active+clean; 283 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 584 KiB/s wr, 6 op/s; 58/624 objects degraded (9.295%) 2024-09-21T09:59:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:07 smithi033 ceph-mon[96568]: Health check failed: Degraded data redundancy: 58/624 objects degraded (9.295%), 8 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:07.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:07 smithi033 ceph-mon[96568]: osdmap e517: 8 total, 7 up, 8 in 2024-09-21T09:59:08.250 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:08 smithi105 podman[101744]: 2024-09-21 09:59:08.157174781 +0000 UTC m=+0.345339680 container init 132bc834f24d6afd074a39b8c8791c6833709b42389d6c9e312aaca69f98941a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-62da54b) 2024-09-21T09:59:08.250 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:08 smithi105 podman[101744]: 2024-09-21 09:59:08.161724873 +0000 UTC m=+0.349889753 container start 132bc834f24d6afd074a39b8c8791c6833709b42389d6c9e312aaca69f98941a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=main-62da54b, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T09:59:08.250 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:08 smithi105 ceph-osd[101782]: -- 172.21.15.105:0/3211223098 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55be756b5a40 con 0x55be76484400 2024-09-21T09:59:08.505 INFO:tasks.rados.rados.0.smithi033.stdout:2464: done (2 left) 2024-09-21T09:59:08.505 INFO:tasks.rados.rados.0.smithi033.stdout:2465: done (1 left) 2024-09-21T09:59:08.505 INFO:tasks.rados.rados.0.smithi033.stdout:2467: done (0 left) 2024-09-21T09:59:08.505 INFO:tasks.rados.rados.0.smithi033.stdout:2468: snap_create 2024-09-21T09:59:08.516 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:08 smithi105 bash[101744]: 132bc834f24d6afd074a39b8c8791c6833709b42389d6c9e312aaca69f98941a 2024-09-21T09:59:08.516 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:08 smithi105 systemd[1]: Started Ceph osd.6 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:59:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:08 smithi105 ceph-mon[86649]: osdmap e518: 8 total, 7 up, 8 in 2024-09-21T09:59:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:08.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:08 smithi105 ceph-mon[86649]: osdmap e519: 8 total, 7 up, 8 in 2024-09-21T09:59:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[98546]: osdmap e518: 8 total, 7 up, 8 in 2024-09-21T09:59:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:08.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[98546]: osdmap e519: 8 total, 7 up, 8 in 2024-09-21T09:59:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[96568]: osdmap e518: 8 total, 7 up, 8 in 2024-09-21T09:59:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:08 smithi033 ceph-mon[96568]: osdmap e519: 8 total, 7 up, 8 in 2024-09-21T09:59:09.502 INFO:tasks.rados.rados.0.smithi033.stdout:2468: done (0 left) 2024-09-21T09:59:09.503 INFO:tasks.rados.rados.0.smithi033.stdout:2469: delete oid 49 current snap is 232 2024-09-21T09:59:09.506 INFO:tasks.rados.rados.0.smithi033.stdout:2469: done (0 left) 2024-09-21T09:59:09.506 INFO:tasks.rados.rados.0.smithi033.stdout:2470: delete oid 41 current snap is 232 2024-09-21T09:59:09.511 INFO:tasks.rados.rados.0.smithi033.stdout:2470: done (0 left) 2024-09-21T09:59:09.511 INFO:tasks.rados.rados.0.smithi033.stdout:2471: copy_from oid 45 from oid 18 current snap is 232 2024-09-21T09:59:09.511 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 232 2024-09-21T09:59:09.511 INFO:tasks.rados.rados.0.smithi033.stdout:2472: seq_num 791 ranges {819200=450560,1269760=475136,1744896=327680} 2024-09-21T09:59:09.518 INFO:tasks.rados.rados.0.smithi033.stdout:2472: writing smithi03385152-6 from 819200 to 1269760 tid 1 2024-09-21T09:59:09.522 INFO:tasks.rados.rados.0.smithi033.stdout:2472: writing smithi03385152-6 from 1269760 to 1744896 tid 2 2024-09-21T09:59:09.524 INFO:tasks.rados.rados.0.smithi033.stdout:2472: writing smithi03385152-6 from 1744896 to 2072576 tid 3 2024-09-21T09:59:09.524 INFO:tasks.rados.rados.0.smithi033.stdout:2473: copy_from oid 41 from oid 11 current snap is 232 2024-09-21T09:59:09.524 INFO:tasks.rados.rados.0.smithi033.stdout:2472: finishing write tid 1 to smithi03385152-6 2024-09-21T09:59:09.524 INFO:tasks.rados.rados.0.smithi033.stdout:2474: setattr oid 37 current snap is 232 2024-09-21T09:59:09.524 INFO:tasks.rados.rados.0.smithi033.stdout:2471: finishing copy_from racing read to smithi03385152-45 2024-09-21T09:59:09.525 INFO:tasks.rados.rados.0.smithi033.stdout:2471: finishing copy_from to smithi03385152-45 2024-09-21T09:59:09.525 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1006 (ObjNum 767 snap 219 seq_num 767) dirty exists 2024-09-21T09:59:09.525 INFO:tasks.rados.rados.0.smithi033.stdout:2473: finishing copy_from to smithi03385152-41 2024-09-21T09:59:09.525 INFO:tasks.rados.rados.0.smithi033.stdout:2473: got expected ENOENT (src dne) 2024-09-21T09:59:09.525 INFO:tasks.rados.rados.0.smithi033.stdout:2473: finishing copy_from racing read to smithi03385152-41 2024-09-21T09:59:09.526 INFO:tasks.rados.rados.0.smithi033.stdout:2471: done (3 left) 2024-09-21T09:59:09.526 INFO:tasks.rados.rados.0.smithi033.stdout:2473: done (2 left) 2024-09-21T09:59:09.526 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 232 2024-09-21T09:59:09.526 INFO:tasks.rados.rados.0.smithi033.stdout:2475: seq_num 792 ranges {851968=466944,1318912=761856,2080768=40960} 2024-09-21T09:59:09.533 INFO:tasks.rados.rados.0.smithi033.stdout:2475: writing smithi03385152-45 from 851968 to 1318912 tid 1 2024-09-21T09:59:09.537 INFO:tasks.rados.rados.0.smithi033.stdout:2475: writing smithi03385152-45 from 1318912 to 2080768 tid 2 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2475: writing smithi03385152-45 from 2080768 to 2121728 tid 3 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2476: rollback oid 14 current snap is 232 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 14 to 196 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2472: finishing write tid 2 to smithi03385152-6 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2472: finishing write tid 3 to smithi03385152-6 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2472: finishing write tid 4 to smithi03385152-6 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2472: finishing write tid 5 to smithi03385152-6 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1599 (ObjNum 791 snap 232 seq_num 791) dirty exists 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2472: left oid 6 (ObjNum 791 snap 232 seq_num 791) 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1044 (ObjNum 731 snap 210 seq_num 731) dirty exists 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2475: finishing write tid 1 to smithi03385152-45 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2472: done (3 left) 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2474: done (2 left) 2024-09-21T09:59:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:2477: setattr oid 22 current snap is 232 2024-09-21T09:59:09.540 INFO:tasks.rados.rados.0.smithi033.stdout:2478: copy_from oid 19 from oid 4 current snap is 232 2024-09-21T09:59:09.541 INFO:tasks.rados.rados.0.smithi033.stdout:2479: rmattr oid 6 current snap is 232 2024-09-21T09:59:09.541 INFO:tasks.rados.rados.0.smithi033.stdout:2479: done (4 left) 2024-09-21T09:59:09.541 INFO:tasks.rados.rados.0.smithi033.stdout:2480: copy_from oid 31 from oid 48 current snap is 232 2024-09-21T09:59:09.541 INFO:tasks.rados.rados.0.smithi033.stdout:2481: read oid 43 snap -1 2024-09-21T09:59:09.541 INFO:tasks.rados.rados.0.smithi033.stdout:2481: expect deleted 2024-09-21T09:59:09.541 INFO:tasks.rados.rados.0.smithi033.stdout:2482: delete oid 10 current snap is 232 2024-09-21T09:59:09.542 INFO:tasks.rados.rados.0.smithi033.stdout:2475: finishing write tid 2 to smithi03385152-45 2024-09-21T09:59:09.542 INFO:tasks.rados.rados.0.smithi033.stdout:2475: finishing write tid 3 to smithi03385152-45 2024-09-21T09:59:09.544 INFO:tasks.rados.rados.0.smithi033.stdout:2481: done (6 left) 2024-09-21T09:59:09.544 INFO:tasks.rados.rados.0.smithi033.stdout:2482: done (5 left) 2024-09-21T09:59:09.544 INFO:tasks.rados.rados.0.smithi033.stdout:2483: rmattr oid 44 current snap is 232 2024-09-21T09:59:09.544 INFO:tasks.rados.rados.0.smithi033.stdout:2483: done (5 left) 2024-09-21T09:59:09.544 INFO:tasks.rados.rados.0.smithi033.stdout:2484: delete oid 5 current snap is 232 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2484: done (5 left) 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2485: rollback oid 32 current snap is 232 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 32 to 210 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2486: rollback oid 28 current snap is 232 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 223 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2487: read oid 50 snap 229 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2487: expect deleted 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2488: read oid 26 snap 220 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2488: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2489: rollback oid 47 current snap is 232 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 230 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2490: rollback oid 15 current snap is 232 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 15 to 202 2024-09-21T09:59:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:2491: snap_create 2024-09-21T09:59:09.549 INFO:tasks.rados.rados.0.smithi033.stdout:2475: finishing write tid 4 to smithi03385152-45 2024-09-21T09:59:09.549 INFO:tasks.rados.rados.0.smithi033.stdout:2489: finishing rollback tid 1 to smithi03385152-47 2024-09-21T09:59:09.550 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1478 (ObjNum 33 snap 0 seq_num 72668536) dirty dne 2024-09-21T09:59:09.551 INFO:tasks.rados.rados.0.smithi033.stdout:2485: finishing rollback tid 1 to smithi03385152-32 2024-09-21T09:59:09.551 INFO:tasks.rados.rados.0.smithi033.stdout:2486: finishing rollback tid 0 to smithi03385152-28 2024-09-21T09:59:09.551 INFO:tasks.rados.rados.0.smithi033.stdout:2476: finishing rollback tid 1 to smithi03385152-14 2024-09-21T09:59:09.551 INFO:tasks.rados.rados.0.smithi033.stdout:2476: finishing rollback tid 2 to smithi03385152-14 2024-09-21T09:59:09.551 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1311 (ObjNum 666 snap 185 seq_num 666) dirty exists 2024-09-21T09:59:09.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1313 (ObjNum 161 snap 0 seq_num 0) dirty exists 2024-09-21T09:59:09.552 INFO:tasks.rados.rados.0.smithi033.stdout:2485: finishing rollback tid 2 to smithi03385152-32 2024-09-21T09:59:09.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 228 (ObjNum 724 snap 205 seq_num 724) dirty exists 2024-09-21T09:59:09.552 INFO:tasks.rados.rados.0.smithi033.stdout:2486: finishing rollback tid 1 to smithi03385152-28 2024-09-21T09:59:09.552 INFO:tasks.rados.rados.0.smithi033.stdout:2486: finishing rollback tid 2 to smithi03385152-28 2024-09-21T09:59:09.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 888 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:59:09.552 INFO:tasks.rados.rados.0.smithi033.stdout:2488: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:09.563 INFO:tasks.rados.rados.0.smithi033.stdout:2490: finishing rollback tid 0 to smithi03385152-15 2024-09-21T09:59:09.563 INFO:tasks.rados.rados.0.smithi033.stdout:2475: finishing write tid 5 to smithi03385152-45 2024-09-21T09:59:09.563 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1309 (ObjNum 792 snap 232 seq_num 792) dirty exists 2024-09-21T09:59:09.563 INFO:tasks.rados.rados.0.smithi033.stdout:2475: left oid 45 (ObjNum 792 snap 232 seq_num 792) 2024-09-21T09:59:09.563 INFO:tasks.rados.rados.0.smithi033.stdout:2490: finishing rollback tid 1 to smithi03385152-15 2024-09-21T09:59:09.563 INFO:tasks.rados.rados.0.smithi033.stdout:2490: finishing rollback tid 2 to smithi03385152-15 2024-09-21T09:59:09.563 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 934 (ObjNum 684 snap 188 seq_num 684) dirty exists 2024-09-21T09:59:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:2480: finishing copy_from racing read to smithi03385152-31 2024-09-21T09:59:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:2480: finishing copy_from to smithi03385152-31 2024-09-21T09:59:09.567 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1271 (ObjNum 770 snap 220 seq_num 770) dirty exists 2024-09-21T09:59:09.573 INFO:tasks.rados.rados.0.smithi033.stdout:2478: finishing copy_from racing read to smithi03385152-19 2024-09-21T09:59:09.576 INFO:tasks.rados.rados.0.smithi033.stdout:2478: finishing copy_from to smithi03385152-19 2024-09-21T09:59:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1046 (ObjNum 746 snap 216 seq_num 746) dirty exists 2024-09-21T09:59:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:09 smithi105 ceph-mon[86649]: pgmap v408: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 284 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 842 KiB/s wr, 8 op/s; 62/627 objects degraded (9.888%) 2024-09-21T09:59:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:09 smithi105 ceph-mon[86649]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:59:09.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:09 smithi105 ceph-mon[86649]: osdmap e520: 8 total, 7 up, 8 in 2024-09-21T09:59:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:09 smithi033 ceph-mon[96568]: pgmap v408: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 284 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 842 KiB/s wr, 8 op/s; 62/627 objects degraded (9.888%) 2024-09-21T09:59:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:09 smithi033 ceph-mon[96568]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:59:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:09 smithi033 ceph-mon[96568]: osdmap e520: 8 total, 7 up, 8 in 2024-09-21T09:59:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:09 smithi033 ceph-mon[98546]: pgmap v408: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 284 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 842 KiB/s wr, 8 op/s; 62/627 objects degraded (9.888%) 2024-09-21T09:59:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:09 smithi033 ceph-mon[98546]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-21T09:59:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:09 smithi033 ceph-mon[98546]: osdmap e520: 8 total, 7 up, 8 in 2024-09-21T09:59:10.061 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:09 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T09:59:09.801+0000 7efdc5e6c640 -1 Falling back to public interface 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2475: done (11 left) 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2476: done (10 left) 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2477: done (9 left) 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2478: done (8 left) 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2480: done (7 left) 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2485: done (6 left) 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2486: done (5 left) 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2487: done (4 left) 2024-09-21T09:59:10.529 INFO:tasks.rados.rados.0.smithi033.stdout:2488: done (3 left) 2024-09-21T09:59:10.530 INFO:tasks.rados.rados.0.smithi033.stdout:2489: done (2 left) 2024-09-21T09:59:10.530 INFO:tasks.rados.rados.0.smithi033.stdout:2490: done (1 left) 2024-09-21T09:59:10.530 INFO:tasks.rados.rados.0.smithi033.stdout:2491: done (0 left) 2024-09-21T09:59:10.530 INFO:tasks.rados.rados.0.smithi033.stdout:2492: copy_from oid 12 from oid 43 current snap is 233 2024-09-21T09:59:10.530 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 233 2024-09-21T09:59:10.530 INFO:tasks.rados.rados.0.smithi033.stdout:2493: seq_num 795 ranges {0=524288,524288=417792,942080=385024} 2024-09-21T09:59:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:59:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:59:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:10.531 INFO:tasks.rados.rados.0.smithi033.stdout:2493: writing smithi03385152-49 from 0 to 524288 tid 1 2024-09-21T09:59:10.534 INFO:tasks.rados.rados.0.smithi033.stdout:2493: writing smithi03385152-49 from 524288 to 942080 tid 2 2024-09-21T09:59:10.537 INFO:tasks.rados.rados.0.smithi033.stdout:2493: writing smithi03385152-49 from 942080 to 1327104 tid 3 2024-09-21T09:59:10.537 INFO:tasks.rados.rados.0.smithi033.stdout:2494: copy_from oid 6 from oid 21 current snap is 233 2024-09-21T09:59:10.537 INFO:tasks.rados.rados.0.smithi033.stdout:2492: finishing copy_from to smithi03385152-12 2024-09-21T09:59:10.537 INFO:tasks.rados.rados.0.smithi033.stdout:2492: got expected ENOENT (src dne) 2024-09-21T09:59:10.537 INFO:tasks.rados.rados.0.smithi033.stdout:2495: snap_remove snap 202 2024-09-21T09:59:11.538 INFO:tasks.rados.rados.0.smithi033.stdout:2495: done (3 left) 2024-09-21T09:59:11.538 INFO:tasks.rados.rados.0.smithi033.stdout:2496: snap_remove snap 227 2024-09-21T09:59:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:11 smithi105 ceph-mon[86649]: pgmap v411: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 285 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 540 KiB/s wr, 10 op/s; 62/633 objects degraded (9.795%) 2024-09-21T09:59:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:11 smithi105 ceph-mon[86649]: osdmap e521: 8 total, 7 up, 8 in 2024-09-21T09:59:11.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:59:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:59:11] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.51.0" 2024-09-21T09:59:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:11 smithi033 ceph-mon[98546]: pgmap v411: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 285 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 540 KiB/s wr, 10 op/s; 62/633 objects degraded (9.795%) 2024-09-21T09:59:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:11 smithi033 ceph-mon[98546]: osdmap e521: 8 total, 7 up, 8 in 2024-09-21T09:59:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:11 smithi033 ceph-mon[96568]: pgmap v411: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 285 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 540 KiB/s wr, 10 op/s; 62/633 objects degraded (9.795%) 2024-09-21T09:59:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:11 smithi033 ceph-mon[96568]: osdmap e521: 8 total, 7 up, 8 in 2024-09-21T09:59:12.548 INFO:tasks.rados.rados.0.smithi033.stdout:2496: done (3 left) 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2497: read oid 20 snap 215 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2497: expect (ObjNum 91939952 snap 21955 seq_num 91851824) 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2498: rollback oid 29 current snap is 233 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 29 to 223 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2492: finishing copy_from racing read to smithi03385152-12 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2493: finishing write tid 1 to smithi03385152-49 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2493: finishing write tid 2 to smithi03385152-49 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2493: finishing write tid 3 to smithi03385152-49 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2493: finishing write tid 4 to smithi03385152-49 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2493: finishing write tid 5 to smithi03385152-49 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1485 (ObjNum 795 snap 233 seq_num 795) dirty exists 2024-09-21T09:59:12.549 INFO:tasks.rados.rados.0.smithi033.stdout:2493: left oid 49 (ObjNum 795 snap 233 seq_num 795) 2024-09-21T09:59:12.550 INFO:tasks.rados.rados.0.smithi033.stdout:2494: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:59:12.550 INFO:tasks.rados.rados.0.smithi033.stdout:2494: finishing copy_from to smithi03385152-6 2024-09-21T09:59:12.550 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1484 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T09:59:12.550 INFO:tasks.rados.rados.0.smithi033.stdout:2492: done (4 left) 2024-09-21T09:59:12.550 INFO:tasks.rados.rados.0.smithi033.stdout:2493: done (3 left) 2024-09-21T09:59:12.550 INFO:tasks.rados.rados.0.smithi033.stdout:2494: done (2 left) 2024-09-21T09:59:12.550 INFO:tasks.rados.rados.0.smithi033.stdout:2499: copy_from oid 1 from oid 18 current snap is 233 2024-09-21T09:59:12.550 INFO:tasks.rados.rados.0.smithi033.stdout:2500: delete oid 23 current snap is 233 2024-09-21T09:59:12.554 INFO:tasks.rados.rados.0.smithi033.stdout:2497: expect (ObjNum 91939952 snap 21955 seq_num 91851824) 2024-09-21T09:59:12.556 INFO:tasks.rados.rados.0.smithi033.stdout:2497: done (3 left) 2024-09-21T09:59:12.556 INFO:tasks.rados.rados.0.smithi033.stdout:2500: done (2 left) 2024-09-21T09:59:12.556 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 233 2024-09-21T09:59:12.557 INFO:tasks.rados.rados.0.smithi033.stdout:2501: seq_num 797 ranges {4284416=557056,4841472=589824,5431296=212992} 2024-09-21T09:59:12.583 INFO:tasks.rados.rados.0.smithi033.stdout:2501: writing smithi03385152-12 from 4284416 to 4841472 tid 1 2024-09-21T09:59:12.587 INFO:tasks.rados.rados.0.smithi033.stdout:2501: writing smithi03385152-12 from 4841472 to 5431296 tid 2 2024-09-21T09:59:12.588 INFO:tasks.rados.rados.0.smithi033.stdout:2501: writing smithi03385152-12 from 5431296 to 5644288 tid 3 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 233 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:2499: finishing copy_from racing read to smithi03385152-1 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:2498: finishing rollback tid 0 to smithi03385152-29 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:2498: finishing rollback tid 1 to smithi03385152-29 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:2499: finishing copy_from to smithi03385152-1 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1006 (ObjNum 767 snap 219 seq_num 767) dirty exists 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:2498: finishing rollback tid 2 to smithi03385152-29 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1641 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:2501: finishing write tid 1 to smithi03385152-12 2024-09-21T09:59:12.589 INFO:tasks.rados.rados.0.smithi033.stdout:2502: seq_num 798 ranges {1155072=573440,1728512=466944,2195456=335872} 2024-09-21T09:59:12.598 INFO:tasks.rados.rados.0.smithi033.stdout:2502: writing smithi03385152-39 from 1155072 to 1728512 tid 1 2024-09-21T09:59:12.601 INFO:tasks.rados.rados.0.smithi033.stdout:2502: writing smithi03385152-39 from 1728512 to 2195456 tid 2 2024-09-21T09:59:12.603 INFO:tasks.rados.rados.0.smithi033.stdout:2502: writing smithi03385152-39 from 2195456 to 2531328 tid 3 2024-09-21T09:59:12.603 INFO:tasks.rados.rados.0.smithi033.stdout:2498: done (3 left) 2024-09-21T09:59:12.603 INFO:tasks.rados.rados.0.smithi033.stdout:2499: done (2 left) 2024-09-21T09:59:12.603 INFO:tasks.rados.rados.0.smithi033.stdout:2503: snap_remove snap 224 2024-09-21T09:59:12.619 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:12 smithi105 ceph-mon[86649]: osdmap e522: 8 total, 7 up, 8 in 2024-09-21T09:59:12.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:12 smithi033 ceph-mon[98546]: osdmap e522: 8 total, 7 up, 8 in 2024-09-21T09:59:12.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:12 smithi033 ceph-mon[96568]: osdmap e522: 8 total, 7 up, 8 in 2024-09-21T09:59:13.540 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T09:59:13.282+0000 7efdc5e6c640 -1 osd.6 512 log_to_monitors true 2024-09-21T09:59:13.551 INFO:tasks.rados.rados.0.smithi033.stdout:2501: finishing write tid 2 to smithi03385152-12 2024-09-21T09:59:13.551 INFO:tasks.rados.rados.0.smithi033.stdout:2501: finishing write tid 3 to smithi03385152-12 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2501: finishing write tid 4 to smithi03385152-12 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2501: finishing write tid 5 to smithi03385152-12 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1489 (ObjNum 797 snap 233 seq_num 797) dirty exists 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2501: left oid 12 (ObjNum 797 snap 233 seq_num 797) 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2502: finishing write tid 1 to smithi03385152-39 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2502: finishing write tid 2 to smithi03385152-39 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2502: finishing write tid 3 to smithi03385152-39 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2502: finishing write tid 4 to smithi03385152-39 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2502: finishing write tid 5 to smithi03385152-39 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1020 (ObjNum 798 snap 233 seq_num 798) dirty exists 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2502: left oid 39 (ObjNum 798 snap 233 seq_num 798) 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2501: done (2 left) 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2502: done (1 left) 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2503: done (0 left) 2024-09-21T09:59:13.552 INFO:tasks.rados.rados.0.smithi033.stdout:2504: setattr oid 27 current snap is 233 2024-09-21T09:59:13.554 INFO:tasks.rados.rados.0.smithi033.stdout:2505: rmattr oid 39 current snap is 233 2024-09-21T09:59:13.554 INFO:tasks.rados.rados.0.smithi033.stdout:2505: done (1 left) 2024-09-21T09:59:13.554 INFO:tasks.rados.rados.0.smithi033.stdout:2506: rollback oid 22 current snap is 233 2024-09-21T09:59:13.554 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 22 to 207 2024-09-21T09:59:13.554 INFO:tasks.rados.rados.0.smithi033.stdout:2507: snap_remove snap 215 2024-09-21T09:59:13.810 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: pgmap v414: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.7 MiB/s wr, 25 op/s; 65/657 objects degraded (9.893%) 2024-09-21T09:59:13.810 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 65/657 objects degraded (9.893%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:13.810 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: osdmap e523: 8 total, 7 up, 8 in 2024-09-21T09:59:13.810 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.811 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.811 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: from='osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:59:13.811 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:59:13.811 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.811 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:13 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: pgmap v414: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.7 MiB/s wr, 25 op/s; 65/657 objects degraded (9.893%) 2024-09-21T09:59:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 65/657 objects degraded (9.893%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: osdmap e523: 8 total, 7 up, 8 in 2024-09-21T09:59:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: from='osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: pgmap v414: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 301 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 2.7 MiB/s wr, 25 op/s; 65/657 objects degraded (9.893%) 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 65/657 objects degraded (9.893%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: osdmap e523: 8 total, 7 up, 8 in 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: from='osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017]' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]} : dispatch 2024-09-21T09:59:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:13.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:13 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:14.176 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 09:59:14 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T09:59:14.011+0000 7efdb9c0e640 -1 osd.6 512 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:59:14.560 INFO:tasks.rados.rados.0.smithi033.stdout:2507: done (2 left) 2024-09-21T09:59:14.561 INFO:tasks.rados.rados.0.smithi033.stdout:2508: delete oid 18 current snap is 233 2024-09-21T09:59:14.561 INFO:tasks.rados.rados.0.smithi033.stdout:2506: finishing rollback tid 0 to smithi03385152-22 2024-09-21T09:59:14.561 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1600 (ObjNum 697 snap 193 seq_num 697) dirty exists 2024-09-21T09:59:14.561 INFO:tasks.rados.rados.0.smithi033.stdout:2506: finishing rollback tid 1 to smithi03385152-22 2024-09-21T09:59:14.561 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1316 (ObjNum 65 snap 0 seq_num 1348685675) dirty dne 2024-09-21T09:59:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:2504: done (2 left) 2024-09-21T09:59:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:2506: done (1 left) 2024-09-21T09:59:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:2508: done (0 left) 2024-09-21T09:59:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:2509: read oid 50 snap -1 2024-09-21T09:59:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:2509: expect deleted 2024-09-21T09:59:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:2510: rollback oid 21 current snap is 233 2024-09-21T09:59:14.569 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 221 2024-09-21T09:59:14.570 INFO:tasks.rados.rados.0.smithi033.stdout:2511: delete oid 35 current snap is 233 2024-09-21T09:59:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:2510: finishing rollback tid 0 to smithi03385152-21 2024-09-21T09:59:14.581 INFO:tasks.rados.rados.0.smithi033.stdout:2511: done (2 left) 2024-09-21T09:59:14.581 INFO:tasks.rados.rados.0.smithi033.stdout:2512: delete oid 33 current snap is 233 2024-09-21T09:59:14.583 INFO:tasks.rados.rados.0.smithi033.stdout:2510: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:59:14.583 INFO:tasks.rados.rados.0.smithi033.stdout:2510: finishing rollback tid 2 to smithi03385152-21 2024-09-21T09:59:14.583 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1491 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:59:14.584 INFO:tasks.rados.rados.0.smithi033.stdout:2510: done (2 left) 2024-09-21T09:59:14.584 INFO:tasks.rados.rados.0.smithi033.stdout:2512: done (1 left) 2024-09-21T09:59:14.584 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 233 2024-09-21T09:59:14.584 INFO:tasks.rados.rados.0.smithi033.stdout:2513: seq_num 802 ranges {385024=638976,1024000=458752,1482752=344064} 2024-09-21T09:59:14.590 INFO:tasks.rados.rados.0.smithi033.stdout:2513: writing smithi03385152-6 from 385024 to 1024000 tid 1 2024-09-21T09:59:14.593 INFO:tasks.rados.rados.0.smithi033.stdout:2513: writing smithi03385152-6 from 1024000 to 1482752 tid 2 2024-09-21T09:59:14.595 INFO:tasks.rados.rados.0.smithi033.stdout:2513: writing smithi03385152-6 from 1482752 to 1826816 tid 3 2024-09-21T09:59:14.595 INFO:tasks.rados.rados.0.smithi033.stdout:2514: delete oid 20 current snap is 233 2024-09-21T09:59:14.595 INFO:tasks.rados.rados.0.smithi033.stdout:2513: finishing write tid 1 to smithi03385152-6 2024-09-21T09:59:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:2513: finishing write tid 2 to smithi03385152-6 2024-09-21T09:59:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:2513: finishing write tid 3 to smithi03385152-6 2024-09-21T09:59:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:2513: finishing write tid 4 to smithi03385152-6 2024-09-21T09:59:14.597 INFO:tasks.rados.rados.0.smithi033.stdout:2514: done (2 left) 2024-09-21T09:59:14.598 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 233 2024-09-21T09:59:14.598 INFO:tasks.rados.rados.0.smithi033.stdout:2515: seq_num 804 ranges {5881856=671744,6553600=622592,7176192=180224} 2024-09-21T09:59:14.633 INFO:tasks.rados.rados.0.smithi033.stdout:2515: writing smithi03385152-19 from 5881856 to 6553600 tid 1 2024-09-21T09:59:14.637 INFO:tasks.rados.rados.0.smithi033.stdout:2515: writing smithi03385152-19 from 6553600 to 7176192 tid 2 2024-09-21T09:59:14.638 INFO:tasks.rados.rados.0.smithi033.stdout:2515: writing smithi03385152-19 from 7176192 to 7356416 tid 3 2024-09-21T09:59:14.638 INFO:tasks.rados.rados.0.smithi033.stdout:2516: snap_create 2024-09-21T09:59:14.638 INFO:tasks.rados.rados.0.smithi033.stdout:2513: finishing write tid 5 to smithi03385152-6 2024-09-21T09:59:14.639 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1605 (ObjNum 802 snap 233 seq_num 802) dirty exists 2024-09-21T09:59:14.639 INFO:tasks.rados.rados.0.smithi033.stdout:2513: left oid 6 (ObjNum 802 snap 233 seq_num 802) 2024-09-21T09:59:14.639 INFO:tasks.rados.rados.0.smithi033.stdout:2515: finishing write tid 1 to smithi03385152-19 2024-09-21T09:59:14.640 INFO:tasks.rados.rados.0.smithi033.stdout:2515: finishing write tid 2 to smithi03385152-19 2024-09-21T09:59:14.641 INFO:tasks.rados.rados.0.smithi033.stdout:2515: finishing write tid 3 to smithi03385152-19 2024-09-21T09:59:14.641 INFO:tasks.rados.rados.0.smithi033.stdout:2515: finishing write tid 4 to smithi03385152-19 2024-09-21T09:59:14.642 INFO:tasks.rados.rados.0.smithi033.stdout:2515: finishing write tid 5 to smithi03385152-19 2024-09-21T09:59:14.642 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1050 (ObjNum 804 snap 233 seq_num 804) dirty exists 2024-09-21T09:59:14.642 INFO:tasks.rados.rados.0.smithi033.stdout:2515: left oid 19 (ObjNum 804 snap 233 seq_num 804) 2024-09-21T09:59:14.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[98546]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:59:14.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[98546]: from='osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:14.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[98546]: osdmap e524: 8 total, 7 up, 8 in 2024-09-21T09:59:14.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[98546]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:14.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[98546]: from='osd.6 ' entity='osd.6' 2024-09-21T09:59:14.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[96568]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:59:14.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[96568]: from='osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:14.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[96568]: osdmap e524: 8 total, 7 up, 8 in 2024-09-21T09:59:14.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[96568]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:14.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:14 smithi033 ceph-mon[96568]: from='osd.6 ' entity='osd.6' 2024-09-21T09:59:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:14 smithi105 ceph-mon[86649]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-21T09:59:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:14 smithi105 ceph-mon[86649]: from='osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017]' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:14 smithi105 ceph-mon[86649]: osdmap e524: 8 total, 7 up, 8 in 2024-09-21T09:59:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:14 smithi105 ceph-mon[86649]: from='osd.6 ' entity='osd.6' cmd={"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:15.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:14 smithi105 ceph-mon[86649]: from='osd.6 ' entity='osd.6' 2024-09-21T09:59:15.553 INFO:tasks.rados.rados.0.smithi033.stdout:2513: done (3 left) 2024-09-21T09:59:15.553 INFO:tasks.rados.rados.0.smithi033.stdout:2515: done (2 left) 2024-09-21T09:59:15.553 INFO:tasks.rados.rados.0.smithi033.stdout:2516: done (1 left) 2024-09-21T09:59:15.553 INFO:tasks.rados.rados.0.smithi033.stdout:2517: delete oid 19 current snap is 234 2024-09-21T09:59:15.559 INFO:tasks.rados.rados.0.smithi033.stdout:2517: done (1 left) 2024-09-21T09:59:15.560 INFO:tasks.rados.rados.0.smithi033.stdout:2518: copy_from oid 31 from oid 6 current snap is 234 2024-09-21T09:59:15.560 INFO:tasks.rados.rados.0.smithi033.stdout:2519: delete oid 5 current snap is 234 2024-09-21T09:59:15.562 INFO:tasks.rados.rados.0.smithi033.stdout:2519: done (2 left) 2024-09-21T09:59:15.562 INFO:tasks.rados.rados.0.smithi033.stdout:2520: read oid 24 snap -1 2024-09-21T09:59:15.562 INFO:tasks.rados.rados.0.smithi033.stdout:2520: expect deleted 2024-09-21T09:59:15.562 INFO:tasks.rados.rados.0.smithi033.stdout:2521: read oid 35 snap -1 2024-09-21T09:59:15.563 INFO:tasks.rados.rados.0.smithi033.stdout:2521: expect deleted 2024-09-21T09:59:15.563 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 234 2024-09-21T09:59:15.563 INFO:tasks.rados.rados.0.smithi033.stdout:2522: seq_num 807 ranges {0=720896,720896=729088,1449984=73728} 2024-09-21T09:59:15.567 INFO:tasks.rados.rados.0.smithi033.stdout:2522: writing smithi03385152-20 from 0 to 720896 tid 1 2024-09-21T09:59:15.574 INFO:tasks.rados.rados.0.smithi033.stdout:2522: writing smithi03385152-20 from 720896 to 1449984 tid 2 2024-09-21T09:59:15.574 INFO:tasks.rados.rados.0.smithi033.stdout:2522: writing smithi03385152-20 from 1449984 to 1523712 tid 3 2024-09-21T09:59:15.574 INFO:tasks.rados.rados.0.smithi033.stdout:2523: delete oid 48 current snap is 234 2024-09-21T09:59:15.575 INFO:tasks.rados.rados.0.smithi033.stdout:2518: finishing copy_from racing read to smithi03385152-31 2024-09-21T09:59:15.578 INFO:tasks.rados.rados.0.smithi033.stdout:2522: finishing write tid 1 to smithi03385152-20 2024-09-21T09:59:15.579 INFO:tasks.rados.rados.0.smithi033.stdout:2522: finishing write tid 2 to smithi03385152-20 2024-09-21T09:59:15.579 INFO:tasks.rados.rados.0.smithi033.stdout:2522: finishing write tid 3 to smithi03385152-20 2024-09-21T09:59:15.579 INFO:tasks.rados.rados.0.smithi033.stdout:2522: finishing write tid 4 to smithi03385152-20 2024-09-21T09:59:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:2522: finishing write tid 5 to smithi03385152-20 2024-09-21T09:59:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1496 (ObjNum 807 snap 234 seq_num 807) dirty exists 2024-09-21T09:59:15.581 INFO:tasks.rados.rados.0.smithi033.stdout:2522: left oid 20 (ObjNum 807 snap 234 seq_num 807) 2024-09-21T09:59:15.586 INFO:tasks.rados.rados.0.smithi033.stdout:2518: finishing copy_from to smithi03385152-31 2024-09-21T09:59:15.586 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1605 (ObjNum 802 snap 233 seq_num 802) dirty exists 2024-09-21T09:59:15.587 INFO:tasks.rados.rados.0.smithi033.stdout:2509: done (5 left) 2024-09-21T09:59:15.587 INFO:tasks.rados.rados.0.smithi033.stdout:2518: done (4 left) 2024-09-21T09:59:15.587 INFO:tasks.rados.rados.0.smithi033.stdout:2520: done (3 left) 2024-09-21T09:59:15.587 INFO:tasks.rados.rados.0.smithi033.stdout:2521: done (2 left) 2024-09-21T09:59:15.587 INFO:tasks.rados.rados.0.smithi033.stdout:2522: done (1 left) 2024-09-21T09:59:15.587 INFO:tasks.rados.rados.0.smithi033.stdout:2523: done (0 left) 2024-09-21T09:59:15.587 INFO:tasks.rados.rados.0.smithi033.stdout:2524: snap_remove snap 205 2024-09-21T09:59:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:15 smithi105 ceph-mon[86649]: pgmap v417: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.2 MiB/s wr, 19 op/s; 65/669 objects degraded (9.716%) 2024-09-21T09:59:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:15 smithi105 ceph-mon[86649]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:59:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:15 smithi105 ceph-mon[86649]: osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017] boot 2024-09-21T09:59:15.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:15 smithi105 ceph-mon[86649]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:59:15.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:15 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:59:15.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:15 smithi105 ceph-mon[86649]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:59:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[98546]: pgmap v417: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.2 MiB/s wr, 19 op/s; 65/669 objects degraded (9.716%) 2024-09-21T09:59:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[98546]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:59:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[98546]: osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017] boot 2024-09-21T09:59:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[98546]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:59:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:59:15.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[98546]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:59:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[96568]: pgmap v417: 121 pgs: 34 active+undersized, 9 active+undersized+degraded, 78 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.2 MiB/s wr, 19 op/s; 65/669 objects degraded (9.716%) 2024-09-21T09:59:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[96568]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:59:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[96568]: osd.6 [v2:172.21.15.105:6816/580557017,v1:172.21.15.105:6817/580557017] boot 2024-09-21T09:59:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[96568]: osdmap e525: 8 total, 8 up, 8 in 2024-09-21T09:59:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 6} : dispatch 2024-09-21T09:59:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:15 smithi033 ceph-mon[96568]: osdmap e526: 8 total, 8 up, 8 in 2024-09-21T09:59:16.569 INFO:tasks.rados.rados.0.smithi033.stdout:2524: done (0 left) 2024-09-21T09:59:16.570 INFO:tasks.rados.rados.0.smithi033.stdout:2525: snap_create 2024-09-21T09:59:17.019 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:17.019 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:17.019 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:16 smithi033 ceph-mon[98546]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:59:17.019 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:17.019 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:17.020 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:16 smithi033 ceph-mon[96568]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:59:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:16 smithi105 ceph-mon[86649]: osdmap e527: 8 total, 8 up, 8 in 2024-09-21T09:59:17.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T09:59:17.017+0000 7fc38e955640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T09:59:17.572 INFO:tasks.rados.rados.0.smithi033.stdout:2525: done (0 left) 2024-09-21T09:59:17.572 INFO:tasks.rados.rados.0.smithi033.stdout:2526: snap_remove snap 197 2024-09-21T09:59:18.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: pgmap v420: 121 pgs: 1 active+recovery_wait+degraded, 25 active+undersized, 4 active+undersized+degraded, 91 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.2 MiB/s wr, 15 op/s; 24/699 objects degraded (3.433%); 5.7 MiB/s, 1 objects/s recovering 2024-09-21T09:59:18.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[96568]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: pgmap v420: 121 pgs: 1 active+recovery_wait+degraded, 25 active+undersized, 4 active+undersized+degraded, 91 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.2 MiB/s wr, 15 op/s; 24/699 objects degraded (3.433%); 5.7 MiB/s, 1 objects/s recovering 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:18.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:18.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:18.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:59:18.248 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:17 smithi033 ceph-mon[98546]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: pgmap v420: 121 pgs: 1 active+recovery_wait+degraded, 25 active+undersized, 4 active+undersized+degraded, 91 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.2 MiB/s wr, 15 op/s; 24/699 objects degraded (3.433%); 5.7 MiB/s, 1 objects/s recovering 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:18.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:59:18.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:17 smithi105 ceph-mon[86649]: osdmap e528: 8 total, 8 up, 8 in 2024-09-21T09:59:18.576 INFO:tasks.rados.rados.0.smithi033.stdout:2526: done (0 left) 2024-09-21T09:59:18.576 INFO:tasks.rados.rados.0.smithi033.stdout:2527: setattr oid 43 current snap is 235 2024-09-21T09:59:18.578 INFO:tasks.rados.rados.0.smithi033.stdout:2528: setattr oid 29 current snap is 235 2024-09-21T09:59:18.579 INFO:tasks.rados.rados.0.smithi033.stdout:2529: rmattr oid 12 current snap is 235 2024-09-21T09:59:18.579 INFO:tasks.rados.rados.0.smithi033.stdout:2530: read oid 6 snap -1 2024-09-21T09:59:18.579 INFO:tasks.rados.rados.0.smithi033.stdout:2530: expect (ObjNum 802 snap 233 seq_num 802) 2024-09-21T09:59:18.579 INFO:tasks.rados.rados.0.smithi033.stdout:2531: copy_from oid 42 from oid 26 current snap is 235 2024-09-21T09:59:18.579 INFO:tasks.rados.rados.0.smithi033.stdout:2532: read oid 46 snap 232 2024-09-21T09:59:18.579 INFO:tasks.rados.rados.0.smithi033.stdout:2532: expect deleted 2024-09-21T09:59:18.579 INFO:tasks.rados.rados.0.smithi033.stdout:2533: setattr oid 10 current snap is 235 2024-09-21T09:59:18.580 INFO:tasks.rados.rados.0.smithi033.stdout:2534: delete oid 5 current snap is 235 2024-09-21T09:59:18.588 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1052 (ObjNum 177 snap 0 seq_num 0) dirty exists 2024-09-21T09:59:18.588 INFO:tasks.rados.rados.0.smithi033.stdout:2532: done (7 left) 2024-09-21T09:59:18.589 INFO:tasks.rados.rados.0.smithi033.stdout:2533: done (6 left) 2024-09-21T09:59:18.589 INFO:tasks.rados.rados.0.smithi033.stdout:2534: done (5 left) 2024-09-21T09:59:18.589 INFO:tasks.rados.rados.0.smithi033.stdout:2535: snap_remove snap 221 2024-09-21T09:59:19.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:18 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:59:19.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:18 smithi033 ceph-mon[96568]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T09:59:19.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:18 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 24/699 objects degraded (3.433%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:19.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:18 smithi033 ceph-mon[96568]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:59:19.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:18 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:59:19.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:18 smithi033 ceph-mon[98546]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T09:59:19.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:18 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 24/699 objects degraded (3.433%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:19.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:18 smithi033 ceph-mon[98546]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:59:19.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:18 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:59:19.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:18 smithi105 ceph-mon[86649]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-09-21T09:59:19.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:18 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 24/699 objects degraded (3.433%), 5 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:19.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:18 smithi105 ceph-mon[86649]: osdmap e529: 8 total, 8 up, 8 in 2024-09-21T09:59:19.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1045 (ObjNum 33 snap 0 seq_num 91546256) dirty exists 2024-09-21T09:59:19.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1490 (ObjNum 797 snap 233 seq_num 797) dirty exists 2024-09-21T09:59:19.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1680 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:59:19.576 INFO:tasks.rados.rados.0.smithi033.stdout:2530: expect (ObjNum 802 snap 233 seq_num 802) 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:2531: finishing copy_from racing read to smithi03385152-42 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:2531: finishing copy_from to smithi03385152-42 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 1473 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:2527: done (5 left) 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:2528: done (4 left) 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:2529: done (3 left) 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:2530: done (2 left) 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:2531: done (1 left) 2024-09-21T09:59:19.638 INFO:tasks.rados.rados.0.smithi033.stdout:2535: done (0 left) 2024-09-21T09:59:19.639 INFO:tasks.rados.rados.0.smithi033.stdout:2536: rollback oid 23 current snap is 235 2024-09-21T09:59:19.639 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 23 to 196 2024-09-21T09:59:19.639 INFO:tasks.rados.rados.0.smithi033.stdout:2537: read oid 40 snap -1 2024-09-21T09:59:19.639 INFO:tasks.rados.rados.0.smithi033.stdout:2537: expect deleted 2024-09-21T09:59:19.639 INFO:tasks.rados.rados.0.smithi033.stdout:2538: snap_create 2024-09-21T09:59:19.640 INFO:tasks.rados.rados.0.smithi033.stdout:2536: finishing rollback tid 1 to smithi03385152-23 2024-09-21T09:59:19.640 INFO:tasks.rados.rados.0.smithi033.stdout:2536: finishing rollback tid 2 to smithi03385152-23 2024-09-21T09:59:19.640 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1498 (ObjNum 689 snap 192 seq_num 689) dirty exists 2024-09-21T09:59:20.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:19 smithi105 ceph-mon[86649]: pgmap v423: 121 pgs: 1 active+recovery_wait+degraded, 16 active+undersized, 4 active+undersized+degraded, 100 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 1.7 MiB/s wr, 14 op/s; 24/702 objects degraded (3.419%); 5.7 MiB/s, 1 objects/s recovering 2024-09-21T09:59:20.216 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:19 smithi105 ceph-mon[86649]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:59:20.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:19 smithi033 ceph-mon[98546]: pgmap v423: 121 pgs: 1 active+recovery_wait+degraded, 16 active+undersized, 4 active+undersized+degraded, 100 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 1.7 MiB/s wr, 14 op/s; 24/702 objects degraded (3.419%); 5.7 MiB/s, 1 objects/s recovering 2024-09-21T09:59:20.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:19 smithi033 ceph-mon[98546]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:59:20.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:19 smithi033 ceph-mon[96568]: pgmap v423: 121 pgs: 1 active+recovery_wait+degraded, 16 active+undersized, 4 active+undersized+degraded, 100 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 1.7 MiB/s wr, 14 op/s; 24/702 objects degraded (3.419%); 5.7 MiB/s, 1 objects/s recovering 2024-09-21T09:59:20.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:19 smithi033 ceph-mon[96568]: osdmap e530: 8 total, 8 up, 8 in 2024-09-21T09:59:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:59:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:59:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:20.577 INFO:tasks.rados.rados.0.smithi033.stdout:2536: done (2 left) 2024-09-21T09:59:20.577 INFO:tasks.rados.rados.0.smithi033.stdout:2537: done (1 left) 2024-09-21T09:59:20.577 INFO:tasks.rados.rados.0.smithi033.stdout:2538: done (0 left) 2024-09-21T09:59:20.577 INFO:tasks.rados.rados.0.smithi033.stdout:2539: read oid 10 snap -1 2024-09-21T09:59:20.577 INFO:tasks.rados.rados.0.smithi033.stdout:2539: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:59:20.578 INFO:tasks.rados.rados.0.smithi033.stdout:2540: read oid 48 snap -1 2024-09-21T09:59:20.578 INFO:tasks.rados.rados.0.smithi033.stdout:2540: expect deleted 2024-09-21T09:59:20.578 INFO:tasks.rados.rados.0.smithi033.stdout:2541: snap_create 2024-09-21T09:59:20.580 INFO:tasks.rados.rados.0.smithi033.stdout:2539: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:59:21.580 INFO:tasks.rados.rados.0.smithi033.stdout:2539: done (2 left) 2024-09-21T09:59:21.580 INFO:tasks.rados.rados.0.smithi033.stdout:2540: done (1 left) 2024-09-21T09:59:21.580 INFO:tasks.rados.rados.0.smithi033.stdout:2541: done (0 left) 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:2542: read oid 40 snap 216 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:2542: expect (ObjNum 91059744 snap 21955 seq_num 872432784) 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:2543: rollback oid 45 current snap is 237 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 45 to 228 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:2544: copy_from oid 7 from oid 32 current snap is 237 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:2545: read oid 26 snap -1 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:2545: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:2546: delete oid 35 current snap is 237 2024-09-21T09:59:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:2542: expect (ObjNum 91059744 snap 21955 seq_num 872432784) 2024-09-21T09:59:21.590 INFO:tasks.rados.rados.0.smithi033.stdout:2542: done (4 left) 2024-09-21T09:59:21.590 INFO:tasks.rados.rados.0.smithi033.stdout:2546: done (3 left) 2024-09-21T09:59:21.590 INFO:tasks.rados.rados.0.smithi033.stdout:2547: snap_remove snap 199 2024-09-21T09:59:21.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:59:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:59:21] "GET /metrics HTTP/1.1" 200 36229 "" "Prometheus/2.51.0" 2024-09-21T09:59:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:21 smithi033 ceph-mon[98546]: pgmap v426: 121 pgs: 1 active+recovery_wait+degraded, 120 active+clean; 333 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 568 KiB/s wr, 7 op/s; 7/702 objects degraded (0.997%) 2024-09-21T09:59:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:21 smithi033 ceph-mon[98546]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:59:21.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:21 smithi033 ceph-mon[96568]: pgmap v426: 121 pgs: 1 active+recovery_wait+degraded, 120 active+clean; 333 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 568 KiB/s wr, 7 op/s; 7/702 objects degraded (0.997%) 2024-09-21T09:59:21.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:21 smithi033 ceph-mon[96568]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:59:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:21 smithi105 ceph-mon[86649]: pgmap v426: 121 pgs: 1 active+recovery_wait+degraded, 120 active+clean; 333 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 568 KiB/s wr, 7 op/s; 7/702 objects degraded (0.997%) 2024-09-21T09:59:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:21 smithi105 ceph-mon[86649]: osdmap e531: 8 total, 8 up, 8 in 2024-09-21T09:59:22.443 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:59:22.591 INFO:tasks.rados.rados.0.smithi033.stdout:2547: done (3 left) 2024-09-21T09:59:22.591 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 237 2024-09-21T09:59:22.592 INFO:tasks.rados.rados.0.smithi033.stdout:2548: seq_num 811 ranges {0=786432,786432=663552,1449984=139264} 2024-09-21T09:59:22.597 INFO:tasks.rados.rados.0.smithi033.stdout:2548: writing smithi03385152-35 from 0 to 786432 tid 1 2024-09-21T09:59:22.602 INFO:tasks.rados.rados.0.smithi033.stdout:2548: writing smithi03385152-35 from 786432 to 1449984 tid 2 2024-09-21T09:59:22.603 INFO:tasks.rados.rados.0.smithi033.stdout:2548: writing smithi03385152-35 from 1449984 to 1589248 tid 3 2024-09-21T09:59:22.603 INFO:tasks.rados.rados.0.smithi033.stdout:2549: rollback oid 2 current snap is 237 2024-09-21T09:59:22.604 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 216 2024-09-21T09:59:22.604 INFO:tasks.rados.rados.0.smithi033.stdout:2545: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:2543: finishing rollback tid 0 to smithi03385152-45 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:2543: finishing rollback tid 1 to smithi03385152-45 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1373 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:2544: finishing copy_from racing read to smithi03385152-7 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:2544: finishing copy_from to smithi03385152-7 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 228 (ObjNum 724 snap 205 seq_num 724) dirty exists 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:2549: finishing rollback tid 0 to smithi03385152-2 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:2543: done (4 left) 2024-09-21T09:59:22.622 INFO:tasks.rados.rados.0.smithi033.stdout:2544: done (3 left) 2024-09-21T09:59:22.623 INFO:tasks.rados.rados.0.smithi033.stdout:2545: done (2 left) 2024-09-21T09:59:22.623 INFO:tasks.rados.rados.0.smithi033.stdout:2550: rmattr oid 6 current snap is 237 2024-09-21T09:59:22.623 INFO:tasks.rados.rados.0.smithi033.stdout:2550: done (2 left) 2024-09-21T09:59:22.623 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 237 2024-09-21T09:59:22.623 INFO:tasks.rados.rados.0.smithi033.stdout:2549: finishing rollback tid 1 to smithi03385152-2 2024-09-21T09:59:22.623 INFO:tasks.rados.rados.0.smithi033.stdout:2551: seq_num 812 ranges {0=802816,802816=548864,1351680=253952} 2024-09-21T09:59:22.626 INFO:tasks.rados.rados.0.smithi033.stdout:2551: writing smithi03385152-9 from 0 to 802816 tid 1 2024-09-21T09:59:22.630 INFO:tasks.rados.rados.0.smithi033.stdout:2551: writing smithi03385152-9 from 802816 to 1351680 tid 2 2024-09-21T09:59:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:2551: writing smithi03385152-9 from 1351680 to 1605632 tid 3 2024-09-21T09:59:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:2552: read oid 28 snap -1 2024-09-21T09:59:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:2552: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:59:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:2553: read oid 12 snap -1 2024-09-21T09:59:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:2553: expect (ObjNum 797 snap 233 seq_num 797) 2024-09-21T09:59:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:2554: snap_create 2024-09-21T09:59:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:2549: finishing rollback tid 2 to smithi03385152-2 2024-09-21T09:59:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 935 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T09:59:22.633 INFO:tasks.rados.rados.0.smithi033.stdout:2548: finishing write tid 1 to smithi03385152-35 2024-09-21T09:59:22.633 INFO:tasks.rados.rados.0.smithi033.stdout:2548: finishing write tid 2 to smithi03385152-35 2024-09-21T09:59:22.633 INFO:tasks.rados.rados.0.smithi033.stdout:2548: finishing write tid 3 to smithi03385152-35 2024-09-21T09:59:22.633 INFO:tasks.rados.rados.0.smithi033.stdout:2548: finishing write tid 4 to smithi03385152-35 2024-09-21T09:59:22.633 INFO:tasks.rados.rados.0.smithi033.stdout:2548: finishing write tid 5 to smithi03385152-35 2024-09-21T09:59:22.633 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 1703 (ObjNum 811 snap 237 seq_num 811) dirty exists 2024-09-21T09:59:22.633 INFO:tasks.rados.rados.0.smithi033.stdout:2548: left oid 35 (ObjNum 811 snap 237 seq_num 811) 2024-09-21T09:59:22.633 INFO:tasks.rados.rados.0.smithi033.stdout:2551: finishing write tid 1 to smithi03385152-9 2024-09-21T09:59:22.635 INFO:tasks.rados.rados.0.smithi033.stdout:2551: finishing write tid 2 to smithi03385152-9 2024-09-21T09:59:22.635 INFO:tasks.rados.rados.0.smithi033.stdout:2551: finishing write tid 3 to smithi03385152-9 2024-09-21T09:59:22.636 INFO:tasks.rados.rados.0.smithi033.stdout:2551: finishing write tid 4 to smithi03385152-9 2024-09-21T09:59:22.637 INFO:tasks.rados.rados.0.smithi033.stdout:2551: finishing write tid 5 to smithi03385152-9 2024-09-21T09:59:22.637 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 1477 (ObjNum 812 snap 237 seq_num 812) dirty exists 2024-09-21T09:59:22.637 INFO:tasks.rados.rados.0.smithi033.stdout:2551: left oid 9 (ObjNum 812 snap 237 seq_num 812) 2024-09-21T09:59:22.641 INFO:tasks.rados.rados.0.smithi033.stdout:2552: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:59:22.703 INFO:tasks.rados.rados.0.smithi033.stdout:2553: expect (ObjNum 797 snap 233 seq_num 797) 2024-09-21T09:59:22.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:22 smithi033 ceph-mon[98546]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:59:22.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:22 smithi033 ceph-mon[96568]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:59:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:22 smithi105 ceph-mon[86649]: osdmap e532: 8 total, 8 up, 8 in 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (6m) 112s ago 10m 19.2M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (5m) 10s ago 10m 85.9M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (12m) 112s ago 12m 16.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (12m) 10s ago 12m 21.9M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (5m) 10s ago 15m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (5m) 112s ago 17m 524M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (4m) 112s ago 17m 56.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (4m) 10s ago 16m 51.4M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (4m) 112s ago 15m 47.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (6m) 112s ago 10m 11.8M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:59:23.053 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (5m) 10s ago 10m 10.7M - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (3m) 112s ago 15m 168M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (3m) 112s ago 14m 126M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (2m) 112s ago 14m 66.9M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 76dbf29565ba 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (116s) 112s ago 14m 11.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b85b61817088 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (83s) 10s ago 13m 110M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 01f4f66dbe72 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (48s) 10s ago 13m 117M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4ff1257a8be1 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (15s) 10s ago 12m 11.5M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 132bc834f24d 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (12m) 10s ago 12m 202M 2480M 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 742c27449369 2024-09-21T09:59:23.054 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (5m) 10s ago 10m 51.6M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 1, 2024-09-21T09:59:23.408 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 7 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 3, 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 12 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:59:23.409 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:59:23.505 INFO:tasks.rados.rados.0.smithi033.stdout:2548: done (5 left) 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2549: done (4 left) 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2551: done (3 left) 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2552: done (2 left) 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2553: done (1 left) 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2554: done (0 left) 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2555: copy_from oid 16 from oid 7 current snap is 238 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2556: read oid 14 snap -1 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2556: expect (ObjNum 666 snap 185 seq_num 666) 2024-09-21T09:59:23.506 INFO:tasks.rados.rados.0.smithi033.stdout:2557: delete oid 40 current snap is 238 2024-09-21T09:59:23.509 INFO:tasks.rados.rados.0.smithi033.stdout:2557: done (2 left) 2024-09-21T09:59:23.509 INFO:tasks.rados.rados.0.smithi033.stdout:2558: snap_remove snap 196 2024-09-21T09:59:23.718 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "12/20 daemons upgraded", 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading osd daemons", 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:59:23.719 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[98546]: pgmap v429: 121 pgs: 121 active+clean; 339 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 360 KiB/s wr, 14 op/s; 1.3 MiB/s, 1 objects/s recovering 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[98546]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[98546]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/702 objects degraded (0.997%), 1 pg degraded) 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[98546]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[98546]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/230275897' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[98546]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[96568]: pgmap v429: 121 pgs: 121 active+clean; 339 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 360 KiB/s wr, 14 op/s; 1.3 MiB/s, 1 objects/s recovering 2024-09-21T09:59:23.720 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[96568]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:23.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[96568]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/702 objects degraded (0.997%), 1 pg degraded) 2024-09-21T09:59:23.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T09:59:23.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[96568]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:59:23.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[96568]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:23.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/230275897' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:23.721 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:23 smithi033 ceph-mon[96568]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:59:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:23 smithi105 ceph-mon[86649]: pgmap v429: 121 pgs: 121 active+clean; 339 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 360 KiB/s wr, 14 op/s; 1.3 MiB/s, 1 objects/s recovering 2024-09-21T09:59:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:23 smithi105 ceph-mon[86649]: from='client.44375 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:23 smithi105 ceph-mon[86649]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 7/702 objects degraded (0.997%), 1 pg degraded) 2024-09-21T09:59:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:23 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T09:59:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:23 smithi105 ceph-mon[86649]: osdmap e533: 8 total, 8 up, 8 in 2024-09-21T09:59:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:23 smithi105 ceph-mon[86649]: from='client.34467 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:24.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:23 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/230275897' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:24.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:23 smithi105 ceph-mon[86649]: osdmap e534: 8 total, 8 up, 8 in 2024-09-21T09:59:24.503 INFO:tasks.rados.rados.0.smithi033.stdout:2558: done (2 left) 2024-09-21T09:59:24.504 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 238 2024-09-21T09:59:24.504 INFO:tasks.rados.rados.0.smithi033.stdout:2559: seq_num 814 ranges {245760=442368,688128=770048,1458176=434176} 2024-09-21T09:59:24.508 INFO:tasks.rados.rados.0.smithi033.stdout:2559: writing smithi03385152-37 from 245760 to 688128 tid 1 2024-09-21T09:59:24.513 INFO:tasks.rados.rados.0.smithi033.stdout:2559: writing smithi03385152-37 from 688128 to 1458176 tid 2 2024-09-21T09:59:24.516 INFO:tasks.rados.rados.0.smithi033.stdout:2559: writing smithi03385152-37 from 1458176 to 1892352 tid 3 2024-09-21T09:59:24.516 INFO:tasks.rados.rados.0.smithi033.stdout:2560: snap_remove snap 234 2024-09-21T09:59:24.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:24 smithi033 ceph-mon[96568]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:24.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:24 smithi033 ceph-mon[96568]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:24.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:24 smithi033 ceph-mon[96568]: osdmap e535: 8 total, 8 up, 8 in 2024-09-21T09:59:24.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:24 smithi033 ceph-mon[98546]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:24.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:24 smithi033 ceph-mon[98546]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:24.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:24 smithi033 ceph-mon[98546]: osdmap e535: 8 total, 8 up, 8 in 2024-09-21T09:59:25.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:24 smithi105 ceph-mon[86649]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:25.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:24 smithi105 ceph-mon[86649]: from='client.34485 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:25.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:24 smithi105 ceph-mon[86649]: osdmap e535: 8 total, 8 up, 8 in 2024-09-21T09:59:25.509 INFO:tasks.rados.rados.0.smithi033.stdout:2560: done (3 left) 2024-09-21T09:59:25.510 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 238 2024-09-21T09:59:25.510 INFO:tasks.rados.rados.0.smithi033.stdout:2561: seq_num 815 ranges {3751936=458752,4210688=655360,4866048=409600,5275648=139264} 2024-09-21T09:59:25.532 INFO:tasks.rados.rados.0.smithi033.stdout:2561: writing smithi03385152-13 from 3751936 to 4210688 tid 1 2024-09-21T09:59:25.535 INFO:tasks.rados.rados.0.smithi033.stdout:2561: writing smithi03385152-13 from 4210688 to 4866048 tid 2 2024-09-21T09:59:25.538 INFO:tasks.rados.rados.0.smithi033.stdout:2561: writing smithi03385152-13 from 4866048 to 5275648 tid 3 2024-09-21T09:59:25.539 INFO:tasks.rados.rados.0.smithi033.stdout:2561: writing smithi03385152-13 from 5275648 to 5414912 tid 4 2024-09-21T09:59:25.539 INFO:tasks.rados.rados.0.smithi033.stdout:2562: copy_from oid 34 from oid 12 current snap is 238 2024-09-21T09:59:25.539 INFO:tasks.rados.rados.0.smithi033.stdout:2556: expect (ObjNum 666 snap 185 seq_num 666) 2024-09-21T09:59:25.582 INFO:tasks.rados.rados.0.smithi033.stdout:2555: finishing copy_from racing read to smithi03385152-16 2024-09-21T09:59:25.582 INFO:tasks.rados.rados.0.smithi033.stdout:2555: finishing copy_from to smithi03385152-16 2024-09-21T09:59:25.582 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 228 (ObjNum 724 snap 205 seq_num 724) dirty exists 2024-09-21T09:59:25.582 INFO:tasks.rados.rados.0.smithi033.stdout:2559: finishing write tid 1 to smithi03385152-37 2024-09-21T09:59:25.582 INFO:tasks.rados.rados.0.smithi033.stdout:2555: done (4 left) 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2556: done (3 left) 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2563: rollback oid 21 current snap is 238 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 220 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2559: finishing write tid 2 to smithi03385152-37 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2559: finishing write tid 3 to smithi03385152-37 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2559: finishing write tid 4 to smithi03385152-37 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2559: finishing write tid 5 to smithi03385152-37 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1049 (ObjNum 814 snap 238 seq_num 814) dirty exists 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2559: left oid 37 (ObjNum 814 snap 238 seq_num 814) 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2561: finishing write tid 1 to smithi03385152-13 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2561: finishing write tid 2 to smithi03385152-13 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2561: finishing write tid 3 to smithi03385152-13 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2561: finishing write tid 4 to smithi03385152-13 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2561: finishing write tid 5 to smithi03385152-13 2024-09-21T09:59:25.583 INFO:tasks.rados.rados.0.smithi033.stdout:2561: finishing write tid 6 to smithi03385152-13 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 1734 (ObjNum 815 snap 238 seq_num 815) dirty exists 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:2561: left oid 13 (ObjNum 815 snap 238 seq_num 815) 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:2562: finishing copy_from racing read to smithi03385152-34 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:2562: finishing copy_from to smithi03385152-34 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1490 (ObjNum 797 snap 233 seq_num 797) dirty exists 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:2559: done (3 left) 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:2561: done (2 left) 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:2562: done (1 left) 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:2564: delete oid 13 current snap is 238 2024-09-21T09:59:25.584 INFO:tasks.rados.rados.0.smithi033.stdout:2564: done (1 left) 2024-09-21T09:59:25.585 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 238 2024-09-21T09:59:25.585 INFO:tasks.rados.rados.0.smithi033.stdout:2565: seq_num 817 ranges {0=491520,491520=417792,909312=425984,1335296=360448} 2024-09-21T09:59:25.587 INFO:tasks.rados.rados.0.smithi033.stdout:2565: writing smithi03385152-30 from 0 to 491520 tid 1 2024-09-21T09:59:25.590 INFO:tasks.rados.rados.0.smithi033.stdout:2565: writing smithi03385152-30 from 491520 to 909312 tid 2 2024-09-21T09:59:25.592 INFO:tasks.rados.rados.0.smithi033.stdout:2565: writing smithi03385152-30 from 909312 to 1335296 tid 3 2024-09-21T09:59:25.595 INFO:tasks.rados.rados.0.smithi033.stdout:2565: writing smithi03385152-30 from 1335296 to 1695744 tid 4 2024-09-21T09:59:25.595 INFO:tasks.rados.rados.0.smithi033.stdout:2566: copy_from oid 14 from oid 2 current snap is 238 2024-09-21T09:59:25.595 INFO:tasks.rados.rados.0.smithi033.stdout:2563: finishing rollback tid 0 to smithi03385152-21 2024-09-21T09:59:25.595 INFO:tasks.rados.rados.0.smithi033.stdout:2567: snap_remove snap 225 2024-09-21T09:59:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:25 smithi033 ceph-mon[96568]: pgmap v432: 121 pgs: 121 active+clean; 341 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 486 KiB/s wr, 16 op/s; 1.3 MiB/s, 1 objects/s recovering 2024-09-21T09:59:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:25 smithi033 ceph-mon[96568]: osdmap e536: 8 total, 8 up, 8 in 2024-09-21T09:59:25.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:25 smithi033 ceph-mon[98546]: pgmap v432: 121 pgs: 121 active+clean; 341 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 486 KiB/s wr, 16 op/s; 1.3 MiB/s, 1 objects/s recovering 2024-09-21T09:59:25.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:25 smithi033 ceph-mon[98546]: osdmap e536: 8 total, 8 up, 8 in 2024-09-21T09:59:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:25 smithi105 ceph-mon[86649]: pgmap v432: 121 pgs: 121 active+clean; 341 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 486 KiB/s wr, 16 op/s; 1.3 MiB/s, 1 objects/s recovering 2024-09-21T09:59:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:25 smithi105 ceph-mon[86649]: osdmap e536: 8 total, 8 up, 8 in 2024-09-21T09:59:26.606 INFO:tasks.rados.rados.0.smithi033.stdout:2567: done (3 left) 2024-09-21T09:59:26.606 INFO:tasks.rados.rados.0.smithi033.stdout:2568: read oid 8 snap -1 2024-09-21T09:59:26.606 INFO:tasks.rados.rados.0.smithi033.stdout:2568: expect (ObjNum 779 snap 225 seq_num 779) 2024-09-21T09:59:26.606 INFO:tasks.rados.rados.0.smithi033.stdout:2569: snap_remove snap 232 2024-09-21T09:59:27.608 INFO:tasks.rados.rados.0.smithi033.stdout:2569: done (4 left) 2024-09-21T09:59:27.608 INFO:tasks.rados.rados.0.smithi033.stdout:2570: read oid 27 snap -1 2024-09-21T09:59:27.609 INFO:tasks.rados.rados.0.smithi033.stdout:2570: expect (ObjNum 697 snap 193 seq_num 697) 2024-09-21T09:59:27.609 INFO:tasks.rados.rados.0.smithi033.stdout:2571: read oid 24 snap 230 2024-09-21T09:59:27.609 INFO:tasks.rados.rados.0.smithi033.stdout:2571: expect deleted 2024-09-21T09:59:27.610 INFO:tasks.rados.rados.0.smithi033.stdout:2572: copy_from oid 48 from oid 12 current snap is 238 2024-09-21T09:59:27.610 INFO:tasks.rados.rados.0.smithi033.stdout:2563: finishing rollback tid 1 to smithi03385152-21 2024-09-21T09:59:27.610 INFO:tasks.rados.rados.0.smithi033.stdout:2563: finishing rollback tid 2 to smithi03385152-21 2024-09-21T09:59:27.610 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1501 (ObjNum 680 snap 187 seq_num 680) dirty exists 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2565: finishing write tid 1 to smithi03385152-30 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2565: finishing write tid 2 to smithi03385152-30 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2565: finishing write tid 3 to smithi03385152-30 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2565: finishing write tid 4 to smithi03385152-30 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2565: finishing write tid 5 to smithi03385152-30 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2565: finishing write tid 6 to smithi03385152-30 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 1744 (ObjNum 817 snap 238 seq_num 817) dirty exists 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2565: left oid 30 (ObjNum 817 snap 238 seq_num 817) 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2566: finishing copy_from racing read to smithi03385152-14 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2566: finishing copy_from to smithi03385152-14 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 935 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T09:59:27.611 INFO:tasks.rados.rados.0.smithi033.stdout:2568: expect (ObjNum 779 snap 225 seq_num 779) 2024-09-21T09:59:27.741 INFO:tasks.rados.rados.0.smithi033.stdout:2572: finishing copy_from racing read to smithi03385152-48 2024-09-21T09:59:27.741 INFO:tasks.rados.rados.0.smithi033.stdout:2572: finishing copy_from to smithi03385152-48 2024-09-21T09:59:27.742 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 1490 (ObjNum 797 snap 233 seq_num 797) dirty exists 2024-09-21T09:59:27.742 INFO:tasks.rados.rados.0.smithi033.stdout:2570: expect (ObjNum 697 snap 193 seq_num 697) 2024-09-21T09:59:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:27 smithi033 ceph-mon[98546]: pgmap v435: 121 pgs: 121 active+clean; 370 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.6 MiB/s wr, 21 op/s 2024-09-21T09:59:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:27 smithi033 ceph-mon[98546]: osdmap e537: 8 total, 8 up, 8 in 2024-09-21T09:59:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:27 smithi033 ceph-mon[96568]: pgmap v435: 121 pgs: 121 active+clean; 370 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.6 MiB/s wr, 21 op/s 2024-09-21T09:59:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:27 smithi033 ceph-mon[96568]: osdmap e537: 8 total, 8 up, 8 in 2024-09-21T09:59:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:27 smithi105 ceph-mon[86649]: pgmap v435: 121 pgs: 121 active+clean; 370 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 5.6 MiB/s wr, 21 op/s 2024-09-21T09:59:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:27 smithi105 ceph-mon[86649]: osdmap e537: 8 total, 8 up, 8 in 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2563: done (6 left) 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2565: done (5 left) 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2566: done (4 left) 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2568: done (3 left) 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2570: done (2 left) 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2571: done (1 left) 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2572: done (0 left) 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2573: read oid 14 snap -1 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2573: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 238 2024-09-21T09:59:28.299 INFO:tasks.rados.rados.0.smithi033.stdout:2574: seq_num 818 ranges {0=507904,507904=704512,1212416=499712} 2024-09-21T09:59:28.301 INFO:tasks.rados.rados.0.smithi033.stdout:2574: writing smithi03385152-10 from 0 to 507904 tid 1 2024-09-21T09:59:28.306 INFO:tasks.rados.rados.0.smithi033.stdout:2574: writing smithi03385152-10 from 507904 to 1212416 tid 2 2024-09-21T09:59:28.308 INFO:tasks.rados.rados.0.smithi033.stdout:2574: writing smithi03385152-10 from 1212416 to 1712128 tid 3 2024-09-21T09:59:28.308 INFO:tasks.rados.rados.0.smithi033.stdout:2575: snap_remove snap 210 2024-09-21T09:59:28.502 INFO:tasks.rados.rados.0.smithi033.stdout:2575: done (2 left) 2024-09-21T09:59:28.502 INFO:tasks.rados.rados.0.smithi033.stdout:2576: read oid 27 snap -1 2024-09-21T09:59:28.502 INFO:tasks.rados.rados.0.smithi033.stdout:2576: expect (ObjNum 697 snap 193 seq_num 697) 2024-09-21T09:59:28.502 INFO:tasks.rados.rados.0.smithi033.stdout:2577: rollback oid 40 current snap is 238 2024-09-21T09:59:28.503 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 40 to 223 2024-09-21T09:59:28.503 INFO:tasks.rados.rados.0.smithi033.stdout:2573: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:59:28.529 INFO:tasks.rados.rados.0.smithi033.stdout:2573: done (3 left) 2024-09-21T09:59:28.529 INFO:tasks.rados.rados.0.smithi033.stdout:2578: read oid 2 snap -1 2024-09-21T09:59:28.529 INFO:tasks.rados.rados.0.smithi033.stdout:2578: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:59:28.529 INFO:tasks.rados.rados.0.smithi033.stdout:2579: copy_from oid 21 from oid 45 current snap is 238 2024-09-21T09:59:28.529 INFO:tasks.rados.rados.0.smithi033.stdout:2580: rollback oid 1 current snap is 238 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 230 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2581: setattr oid 39 current snap is 238 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2582: setattr oid 35 current snap is 238 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2574: finishing write tid 1 to smithi03385152-10 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2574: finishing write tid 2 to smithi03385152-10 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2574: finishing write tid 3 to smithi03385152-10 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2574: finishing write tid 4 to smithi03385152-10 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2574: finishing write tid 5 to smithi03385152-10 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1056 (ObjNum 818 snap 238 seq_num 818) dirty exists 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2574: left oid 10 (ObjNum 818 snap 238 seq_num 818) 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2577: finishing rollback tid 1 to smithi03385152-40 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2577: finishing rollback tid 2 to smithi03385152-40 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1479 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T09:59:28.530 INFO:tasks.rados.rados.0.smithi033.stdout:2574: done (7 left) 2024-09-21T09:59:28.531 INFO:tasks.rados.rados.0.smithi033.stdout:2577: done (6 left) 2024-09-21T09:59:28.531 INFO:tasks.rados.rados.0.smithi033.stdout:2583: read oid 26 snap -1 2024-09-21T09:59:28.531 INFO:tasks.rados.rados.0.smithi033.stdout:2583: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:28.531 INFO:tasks.rados.rados.0.smithi033.stdout:2584: rmattr oid 46 current snap is 238 2024-09-21T09:59:28.531 INFO:tasks.rados.rados.0.smithi033.stdout:2584: done (7 left) 2024-09-21T09:59:28.531 INFO:tasks.rados.rados.0.smithi033.stdout:2585: copy_from oid 36 from oid 15 current snap is 238 2024-09-21T09:59:28.531 INFO:tasks.rados.rados.0.smithi033.stdout:2586: delete oid 3 current snap is 238 2024-09-21T09:59:28.532 INFO:tasks.rados.rados.0.smithi033.stdout:2578: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:59:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1021 (ObjNum 798 snap 233 seq_num 798) dirty exists 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2580: finishing rollback tid 0 to smithi03385152-1 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2580: finishing rollback tid 1 to smithi03385152-1 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1008 (ObjNum 65 snap 0 seq_num 1734110568) dirty dne 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 1770 (ObjNum 811 snap 237 seq_num 811) dirty exists 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2579: finishing copy_from to smithi03385152-21 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2579: got expected ENOENT (src dne) 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2579: finishing copy_from racing read to smithi03385152-21 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2578: done (8 left) 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2579: done (7 left) 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2580: done (6 left) 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2581: done (5 left) 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2582: done (4 left) 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2586: done (3 left) 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:2587: rollback oid 5 current snap is 238 2024-09-21T09:59:28.549 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 5 to 223 2024-09-21T09:59:28.550 INFO:tasks.rados.rados.0.smithi033.stdout:2583: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:2585: finishing copy_from racing read to smithi03385152-36 2024-09-21T09:59:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:2587: finishing rollback tid 1 to smithi03385152-5 2024-09-21T09:59:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:2587: finishing rollback tid 2 to smithi03385152-5 2024-09-21T09:59:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 1023 (ObjNum 640 snap 175 seq_num 640) dirty exists 2024-09-21T09:59:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:2576: expect (ObjNum 697 snap 193 seq_num 697) 2024-09-21T09:59:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:28 smithi033 ceph-mon[98546]: osdmap e538: 8 total, 8 up, 8 in 2024-09-21T09:59:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:28 smithi033 ceph-mon[98546]: osdmap e539: 8 total, 8 up, 8 in 2024-09-21T09:59:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:28 smithi033 ceph-mon[96568]: osdmap e538: 8 total, 8 up, 8 in 2024-09-21T09:59:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:28 smithi033 ceph-mon[96568]: osdmap e539: 8 total, 8 up, 8 in 2024-09-21T09:59:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:28 smithi105 ceph-mon[86649]: osdmap e538: 8 total, 8 up, 8 in 2024-09-21T09:59:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:28 smithi105 ceph-mon[86649]: osdmap e539: 8 total, 8 up, 8 in 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:2585: finishing copy_from to smithi03385152-36 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 934 (ObjNum 684 snap 188 seq_num 684) dirty exists 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:2576: done (3 left) 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:2583: done (2 left) 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:2585: done (1 left) 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:2587: done (0 left) 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:2588: read oid 44 snap -1 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:2588: expect deleted 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 238 2024-09-21T09:59:29.118 INFO:tasks.rados.rados.0.smithi033.stdout:2589: seq_num 820 ranges {4300800=540672,4841472=475136,5316608=598016,5914624=131072} 2024-09-21T09:59:29.143 INFO:tasks.rados.rados.0.smithi033.stdout:2589: writing smithi03385152-16 from 4300800 to 4841472 tid 1 2024-09-21T09:59:29.146 INFO:tasks.rados.rados.0.smithi033.stdout:2589: writing smithi03385152-16 from 4841472 to 5316608 tid 2 2024-09-21T09:59:29.149 INFO:tasks.rados.rados.0.smithi033.stdout:2589: writing smithi03385152-16 from 5316608 to 5914624 tid 3 2024-09-21T09:59:29.151 INFO:tasks.rados.rados.0.smithi033.stdout:2589: writing smithi03385152-16 from 5914624 to 6045696 tid 4 2024-09-21T09:59:29.151 INFO:tasks.rados.rados.0.smithi033.stdout:2590: read oid 27 snap -1 2024-09-21T09:59:29.151 INFO:tasks.rados.rados.0.smithi033.stdout:2590: expect (ObjNum 697 snap 193 seq_num 697) 2024-09-21T09:59:29.151 INFO:tasks.rados.rados.0.smithi033.stdout:2591: snap_create 2024-09-21T09:59:29.151 INFO:tasks.rados.rados.0.smithi033.stdout:2589: finishing write tid 1 to smithi03385152-16 2024-09-21T09:59:29.151 INFO:tasks.rados.rados.0.smithi033.stdout:2589: finishing write tid 2 to smithi03385152-16 2024-09-21T09:59:29.155 INFO:tasks.rados.rados.0.smithi033.stdout:2589: finishing write tid 3 to smithi03385152-16 2024-09-21T09:59:29.156 INFO:tasks.rados.rados.0.smithi033.stdout:2589: finishing write tid 4 to smithi03385152-16 2024-09-21T09:59:29.156 INFO:tasks.rados.rados.0.smithi033.stdout:2589: finishing write tid 5 to smithi03385152-16 2024-09-21T09:59:29.157 INFO:tasks.rados.rados.0.smithi033.stdout:2589: finishing write tid 6 to smithi03385152-16 2024-09-21T09:59:29.158 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 1779 (ObjNum 820 snap 238 seq_num 820) dirty exists 2024-09-21T09:59:29.158 INFO:tasks.rados.rados.0.smithi033.stdout:2589: left oid 16 (ObjNum 820 snap 238 seq_num 820) 2024-09-21T09:59:29.189 INFO:tasks.rados.rados.0.smithi033.stdout:2590: expect (ObjNum 697 snap 193 seq_num 697) 2024-09-21T09:59:29.748 INFO:tasks.rados.rados.0.smithi033.stdout:2588: done (3 left) 2024-09-21T09:59:29.748 INFO:tasks.rados.rados.0.smithi033.stdout:2589: done (2 left) 2024-09-21T09:59:29.748 INFO:tasks.rados.rados.0.smithi033.stdout:2590: done (1 left) 2024-09-21T09:59:29.748 INFO:tasks.rados.rados.0.smithi033.stdout:2591: done (0 left) 2024-09-21T09:59:29.748 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 239 2024-09-21T09:59:29.748 INFO:tasks.rados.rados.0.smithi033.stdout:2592: seq_num 821 ranges {2826240=557056,3383296=761856,4145152=442368} 2024-09-21T09:59:29.766 INFO:tasks.rados.rados.0.smithi033.stdout:2592: writing smithi03385152-36 from 2826240 to 3383296 tid 1 2024-09-21T09:59:29.770 INFO:tasks.rados.rados.0.smithi033.stdout:2592: writing smithi03385152-36 from 3383296 to 4145152 tid 2 2024-09-21T09:59:29.772 INFO:tasks.rados.rados.0.smithi033.stdout:2592: writing smithi03385152-36 from 4145152 to 4587520 tid 3 2024-09-21T09:59:29.773 INFO:tasks.rados.rados.0.smithi033.stdout:2593: snap_create 2024-09-21T09:59:29.775 INFO:tasks.rados.rados.0.smithi033.stdout:2592: finishing write tid 1 to smithi03385152-36 2024-09-21T09:59:29.776 INFO:tasks.rados.rados.0.smithi033.stdout:2592: finishing write tid 2 to smithi03385152-36 2024-09-21T09:59:29.776 INFO:tasks.rados.rados.0.smithi033.stdout:2592: finishing write tid 3 to smithi03385152-36 2024-09-21T09:59:29.776 INFO:tasks.rados.rados.0.smithi033.stdout:2592: finishing write tid 4 to smithi03385152-36 2024-09-21T09:59:29.778 INFO:tasks.rados.rados.0.smithi033.stdout:2592: finishing write tid 5 to smithi03385152-36 2024-09-21T09:59:29.778 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1784 (ObjNum 821 snap 239 seq_num 821) dirty exists 2024-09-21T09:59:29.778 INFO:tasks.rados.rados.0.smithi033.stdout:2592: left oid 36 (ObjNum 821 snap 239 seq_num 821) 2024-09-21T09:59:29.932 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:29 smithi105 ceph-mon[86649]: pgmap v438: 121 pgs: 121 active+clean; 370 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.1 MiB/s wr, 13 op/s 2024-09-21T09:59:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:29 smithi033 ceph-mon[98546]: pgmap v438: 121 pgs: 121 active+clean; 370 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.1 MiB/s wr, 13 op/s 2024-09-21T09:59:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:29 smithi033 ceph-mon[96568]: pgmap v438: 121 pgs: 121 active+clean; 370 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.1 MiB/s wr, 13 op/s 2024-09-21T09:59:30.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:59:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:59:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:30.629 INFO:tasks.rados.rados.0.smithi033.stdout:2592: done (1 left) 2024-09-21T09:59:30.629 INFO:tasks.rados.rados.0.smithi033.stdout:2593: done (0 left) 2024-09-21T09:59:30.629 INFO:tasks.rados.rados.0.smithi033.stdout:2594: read oid 5 snap 228 2024-09-21T09:59:30.629 INFO:tasks.rados.rados.0.smithi033.stdout:2594: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:59:30.629 INFO:tasks.rados.rados.0.smithi033.stdout:2595: snap_remove snap 229 2024-09-21T09:59:30.948 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:30 smithi105 ceph-mon[86649]: osdmap e540: 8 total, 8 up, 8 in 2024-09-21T09:59:30.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:30 smithi033 ceph-mon[98546]: osdmap e540: 8 total, 8 up, 8 in 2024-09-21T09:59:30.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:30 smithi033 ceph-mon[96568]: osdmap e540: 8 total, 8 up, 8 in 2024-09-21T09:59:31.634 INFO:tasks.rados.rados.0.smithi033.stdout:2595: done (1 left) 2024-09-21T09:59:31.634 INFO:tasks.rados.rados.0.smithi033.stdout:2596: delete oid 36 current snap is 240 2024-09-21T09:59:31.634 INFO:tasks.rados.rados.0.smithi033.stdout:2594: expect (ObjNum 640 snap 175 seq_num 640) 2024-09-21T09:59:31.701 INFO:tasks.rados.rados.0.smithi033.stdout:2594: done (1 left) 2024-09-21T09:59:31.701 INFO:tasks.rados.rados.0.smithi033.stdout:2596: done (0 left) 2024-09-21T09:59:31.701 INFO:tasks.rados.rados.0.smithi033.stdout:2597: copy_from oid 8 from oid 14 current snap is 240 2024-09-21T09:59:31.701 INFO:tasks.rados.rados.0.smithi033.stdout:2598: snap_remove snap 235 2024-09-21T09:59:31.998 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:59:31] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.51.0" 2024-09-21T09:59:31.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[96568]: pgmap v441: 121 pgs: 121 active+clean; 372 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 418 KiB/s wr, 7 op/s 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[96568]: osdmap e541: 8 total, 8 up, 8 in 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[98546]: pgmap v441: 121 pgs: 121 active+clean; 372 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 418 KiB/s wr, 7 op/s 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[98546]: osdmap e541: 8 total, 8 up, 8 in 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:31.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:31 smithi105 ceph-mon[86649]: pgmap v441: 121 pgs: 121 active+clean; 372 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 418 KiB/s wr, 7 op/s 2024-09-21T09:59:32.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:31 smithi105 ceph-mon[86649]: osdmap e541: 8 total, 8 up, 8 in 2024-09-21T09:59:32.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:32.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:32.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:32.653 INFO:tasks.rados.rados.0.smithi033.stdout:2598: done (1 left) 2024-09-21T09:59:32.653 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 240 2024-09-21T09:59:32.653 INFO:tasks.rados.rados.0.smithi033.stdout:2599: seq_num 823 ranges {1826816=589824,2416640=573440,2990080=630784} 2024-09-21T09:59:32.666 INFO:tasks.rados.rados.0.smithi033.stdout:2599: writing smithi03385152-31 from 1826816 to 2416640 tid 1 2024-09-21T09:59:32.669 INFO:tasks.rados.rados.0.smithi033.stdout:2599: writing smithi03385152-31 from 2416640 to 2990080 tid 2 2024-09-21T09:59:32.674 INFO:tasks.rados.rados.0.smithi033.stdout:2599: writing smithi03385152-31 from 2990080 to 3620864 tid 3 2024-09-21T09:59:32.674 INFO:tasks.rados.rados.0.smithi033.stdout:2600: read oid 25 snap -1 2024-09-21T09:59:32.674 INFO:tasks.rados.rados.0.smithi033.stdout:2600: expect (ObjNum 787 snap 230 seq_num 787) 2024-09-21T09:59:32.674 INFO:tasks.rados.rados.0.smithi033.stdout:2601: rmattr oid 26 current snap is 240 2024-09-21T09:59:32.674 INFO:tasks.rados.rados.0.smithi033.stdout:2597: finishing copy_from racing read to smithi03385152-8 2024-09-21T09:59:32.675 INFO:tasks.rados.rados.0.smithi033.stdout:2602: snap_create 2024-09-21T09:59:32.675 INFO:tasks.rados.rados.0.smithi033.stdout:2597: finishing copy_from to smithi03385152-8 2024-09-21T09:59:32.675 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 935 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T09:59:32.675 INFO:tasks.rados.rados.0.smithi033.stdout:2599: finishing write tid 1 to smithi03385152-31 2024-09-21T09:59:32.675 INFO:tasks.rados.rados.0.smithi033.stdout:2599: finishing write tid 2 to smithi03385152-31 2024-09-21T09:59:32.675 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 0 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T09:59:32.679 INFO:tasks.rados.rados.0.smithi033.stdout:2599: finishing write tid 3 to smithi03385152-31 2024-09-21T09:59:32.679 INFO:tasks.rados.rados.0.smithi033.stdout:2599: finishing write tid 4 to smithi03385152-31 2024-09-21T09:59:32.679 INFO:tasks.rados.rados.0.smithi033.stdout:2599: finishing write tid 5 to smithi03385152-31 2024-09-21T09:59:32.679 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1805 (ObjNum 823 snap 240 seq_num 823) dirty exists 2024-09-21T09:59:32.679 INFO:tasks.rados.rados.0.smithi033.stdout:2599: left oid 31 (ObjNum 823 snap 240 seq_num 823) 2024-09-21T09:59:32.686 INFO:tasks.rados.rados.0.smithi033.stdout:2600: expect (ObjNum 787 snap 230 seq_num 787) 2024-09-21T09:59:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:32 smithi105 ceph-mon[86649]: osdmap e542: 8 total, 8 up, 8 in 2024-09-21T09:59:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:32 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:59:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:32 smithi033 ceph-mon[96568]: osdmap e542: 8 total, 8 up, 8 in 2024-09-21T09:59:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:32 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:59:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:32 smithi033 ceph-mon[98546]: osdmap e542: 8 total, 8 up, 8 in 2024-09-21T09:59:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:32 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16} : dispatch 2024-09-21T09:59:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2597: done (4 left) 2024-09-21T09:59:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2599: done (3 left) 2024-09-21T09:59:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2600: done (2 left) 2024-09-21T09:59:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2601: done (1 left) 2024-09-21T09:59:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2602: done (0 left) 2024-09-21T09:59:33.506 INFO:tasks.rados.rados.0.smithi033.stdout:2603: snap_remove snap 238 2024-09-21T09:59:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:33 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:59:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:33 smithi105 ceph-mon[86649]: Upgrade: osd.7 is safe to restart 2024-09-21T09:59:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:33 smithi105 ceph-mon[86649]: pgmap v444: 121 pgs: 121 active+clean; 383 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-09-21T09:59:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:33 smithi105 ceph-mon[86649]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:59:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:33 smithi105 ceph-mon[86649]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:59:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:33 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:33 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:59:33.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:33 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:59:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[98546]: Upgrade: osd.7 is safe to restart 2024-09-21T09:59:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[98546]: pgmap v444: 121 pgs: 121 active+clean; 383 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-09-21T09:59:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[98546]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:59:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[98546]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[96568]: Upgrade: osd.7 is safe to restart 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[96568]: pgmap v444: 121 pgs: 121 active+clean; 383 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[96568]: osdmap e543: 8 total, 8 up, 8 in 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[96568]: osdmap e544: 8 total, 8 up, 8 in 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-21T09:59:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:33 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:34.506 INFO:tasks.rados.rados.0.smithi033.stdout:2603: done (0 left) 2024-09-21T09:59:34.506 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 241 2024-09-21T09:59:34.506 INFO:tasks.rados.rados.0.smithi033.stdout:2604: seq_num 824 ranges {4300800=606208,4907008=458752,5365760=573440,5939200=172032} 2024-09-21T09:59:34.533 INFO:tasks.rados.rados.0.smithi033.stdout:2604: writing smithi03385152-32 from 4300800 to 4907008 tid 1 2024-09-21T09:59:34.536 INFO:tasks.rados.rados.0.smithi033.stdout:2604: writing smithi03385152-32 from 4907008 to 5365760 tid 2 2024-09-21T09:59:34.540 INFO:tasks.rados.rados.0.smithi033.stdout:2604: writing smithi03385152-32 from 5365760 to 5939200 tid 3 2024-09-21T09:59:34.541 INFO:tasks.rados.rados.0.smithi033.stdout:2604: writing smithi03385152-32 from 5939200 to 6111232 tid 4 2024-09-21T09:59:34.541 INFO:tasks.rados.rados.0.smithi033.stdout:2605: snap_remove snap 173 2024-09-21T09:59:34.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:34 smithi105 ceph-mon[86649]: Upgrade: Updating osd.7 2024-09-21T09:59:34.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:34 smithi105 ceph-mon[86649]: Deploying daemon osd.7 on smithi105 2024-09-21T09:59:34.895 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:34 smithi105 ceph-mon[86649]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:59:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:34 smithi033 ceph-mon[98546]: Upgrade: Updating osd.7 2024-09-21T09:59:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:34 smithi033 ceph-mon[98546]: Deploying daemon osd.7 on smithi105 2024-09-21T09:59:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:34 smithi033 ceph-mon[98546]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:59:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:34 smithi033 ceph-mon[96568]: Upgrade: Updating osd.7 2024-09-21T09:59:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:34 smithi033 ceph-mon[96568]: Deploying daemon osd.7 on smithi105 2024-09-21T09:59:34.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:34 smithi033 ceph-mon[96568]: osdmap e545: 8 total, 8 up, 8 in 2024-09-21T09:59:35.511 INFO:tasks.rados.rados.0.smithi033.stdout:2605: done (1 left) 2024-09-21T09:59:35.511 INFO:tasks.rados.rados.0.smithi033.stdout:2606: copy_from oid 8 from oid 42 current snap is 241 2024-09-21T09:59:35.511 INFO:tasks.rados.rados.0.smithi033.stdout:2604: finishing write tid 1 to smithi03385152-32 2024-09-21T09:59:35.511 INFO:tasks.rados.rados.0.smithi033.stdout:2604: finishing write tid 2 to smithi03385152-32 2024-09-21T09:59:35.511 INFO:tasks.rados.rados.0.smithi033.stdout:2604: finishing write tid 3 to smithi03385152-32 2024-09-21T09:59:35.511 INFO:tasks.rados.rados.0.smithi033.stdout:2604: finishing write tid 4 to smithi03385152-32 2024-09-21T09:59:35.511 INFO:tasks.rados.rados.0.smithi033.stdout:2604: finishing write tid 5 to smithi03385152-32 2024-09-21T09:59:35.511 INFO:tasks.rados.rados.0.smithi033.stdout:2604: finishing write tid 6 to smithi03385152-32 2024-09-21T09:59:35.512 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 250 (ObjNum 824 snap 241 seq_num 824) dirty exists 2024-09-21T09:59:35.512 INFO:tasks.rados.rados.0.smithi033.stdout:2604: left oid 32 (ObjNum 824 snap 241 seq_num 824) 2024-09-21T09:59:35.512 INFO:tasks.rados.rados.0.smithi033.stdout:2604: done (1 left) 2024-09-21T09:59:35.512 INFO:tasks.rados.rados.0.smithi033.stdout:2607: read oid 40 snap 222 2024-09-21T09:59:35.512 INFO:tasks.rados.rados.0.smithi033.stdout:2607: expect (ObjNum 114048848 snap 21955 seq_num 91560512) 2024-09-21T09:59:35.512 INFO:tasks.rados.rados.0.smithi033.stdout:2608: delete oid 3 current snap is 241 2024-09-21T09:59:35.514 INFO:tasks.rados.rados.0.smithi033.stdout:2607: expect (ObjNum 114048848 snap 21955 seq_num 91560512) 2024-09-21T09:59:35.521 INFO:tasks.rados.rados.0.smithi033.stdout:2607: done (2 left) 2024-09-21T09:59:35.521 INFO:tasks.rados.rados.0.smithi033.stdout:2608: done (1 left) 2024-09-21T09:59:35.521 INFO:tasks.rados.rados.0.smithi033.stdout:2609: rmattr oid 15 current snap is 241 2024-09-21T09:59:35.521 INFO:tasks.rados.rados.0.smithi033.stdout:2609: done (1 left) 2024-09-21T09:59:35.521 INFO:tasks.rados.rados.0.smithi033.stdout:2610: read oid 33 snap -1 2024-09-21T09:59:35.521 INFO:tasks.rados.rados.0.smithi033.stdout:2610: expect deleted 2024-09-21T09:59:35.521 INFO:tasks.rados.rados.0.smithi033.stdout:2611: delete oid 28 current snap is 241 2024-09-21T09:59:35.525 INFO:tasks.rados.rados.0.smithi033.stdout:2606: finishing copy_from racing read to smithi03385152-8 2024-09-21T09:59:35.527 INFO:tasks.rados.rados.0.smithi033.stdout:2610: done (2 left) 2024-09-21T09:59:35.527 INFO:tasks.rados.rados.0.smithi033.stdout:2611: done (1 left) 2024-09-21T09:59:35.527 INFO:tasks.rados.rados.0.smithi033.stdout:2612: setattr oid 27 current snap is 241 2024-09-21T09:59:35.527 INFO:tasks.rados.rados.0.smithi033.stdout:2606: finishing copy_from to smithi03385152-8 2024-09-21T09:59:35.527 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 1473 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T09:59:35.529 INFO:tasks.rados.rados.0.smithi033.stdout:2606: done (1 left) 2024-09-21T09:59:35.529 INFO:tasks.rados.rados.0.smithi033.stdout:2613: read oid 14 snap -1 2024-09-21T09:59:35.529 INFO:tasks.rados.rados.0.smithi033.stdout:2613: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:59:35.529 INFO:tasks.rados.rados.0.smithi033.stdout:2614: snap_create 2024-09-21T09:59:35.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1606 (ObjNum 697 snap 193 seq_num 697) dirty exists 2024-09-21T09:59:35.537 INFO:tasks.rados.rados.0.smithi033.stdout:2613: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:59:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:35 smithi105 ceph-mon[86649]: pgmap v447: 121 pgs: 121 active+clean; 383 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-09-21T09:59:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:35 smithi105 ceph-mon[86649]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:59:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:35 smithi033 ceph-mon[98546]: pgmap v447: 121 pgs: 121 active+clean; 383 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-09-21T09:59:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:35 smithi033 ceph-mon[98546]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:59:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:35 smithi033 ceph-mon[96568]: pgmap v447: 121 pgs: 121 active+clean; 383 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 24 MiB/s rd, 2.9 MiB/s wr, 17 op/s 2024-09-21T09:59:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:35 smithi033 ceph-mon[96568]: osdmap e546: 8 total, 8 up, 8 in 2024-09-21T09:59:36.511 INFO:tasks.rados.rados.0.smithi033.stdout:2612: done (2 left) 2024-09-21T09:59:36.511 INFO:tasks.rados.rados.0.smithi033.stdout:2613: done (1 left) 2024-09-21T09:59:36.511 INFO:tasks.rados.rados.0.smithi033.stdout:2614: done (0 left) 2024-09-21T09:59:36.511 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 242 2024-09-21T09:59:36.511 INFO:tasks.rados.rados.0.smithi033.stdout:2615: seq_num 827 ranges {8683520=655360,9338880=516096,9854976=688128} 2024-09-21T09:59:36.563 INFO:tasks.rados.rados.0.smithi033.stdout:2615: writing smithi03385152-23 from 8683520 to 9338880 tid 1 2024-09-21T09:59:36.565 INFO:tasks.rados.rados.0.smithi033.stdout:2615: writing smithi03385152-23 from 9338880 to 9854976 tid 2 2024-09-21T09:59:36.570 INFO:tasks.rados.rados.0.smithi033.stdout:2615: writing smithi03385152-23 from 9854976 to 10543104 tid 3 2024-09-21T09:59:36.570 INFO:tasks.rados.rados.0.smithi033.stdout:2616: snap_create 2024-09-21T09:59:36.570 INFO:tasks.rados.rados.0.smithi033.stdout:2615: finishing write tid 1 to smithi03385152-23 2024-09-21T09:59:36.570 INFO:tasks.rados.rados.0.smithi033.stdout:2615: finishing write tid 2 to smithi03385152-23 2024-09-21T09:59:36.573 INFO:tasks.rados.rados.0.smithi033.stdout:2615: finishing write tid 3 to smithi03385152-23 2024-09-21T09:59:36.573 INFO:tasks.rados.rados.0.smithi033.stdout:2615: finishing write tid 4 to smithi03385152-23 2024-09-21T09:59:36.575 INFO:tasks.rados.rados.0.smithi033.stdout:2615: finishing write tid 5 to smithi03385152-23 2024-09-21T09:59:36.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1505 (ObjNum 827 snap 242 seq_num 827) dirty exists 2024-09-21T09:59:36.575 INFO:tasks.rados.rados.0.smithi033.stdout:2615: left oid 23 (ObjNum 827 snap 242 seq_num 827) 2024-09-21T09:59:36.726 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:36 smithi105 systemd[1]: Stopping Ceph osd.7 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:59:37.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[55164]: 2024-09-21T09:59:36.722+0000 7f5966f86640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:59:37.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[55164]: 2024-09-21T09:59:36.723+0000 7f5966f86640 -1 osd.7 547 *** Got signal Terminated *** 2024-09-21T09:59:37.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[55164]: 2024-09-21T09:59:36.723+0000 7f5966f86640 -1 osd.7 547 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T09:59:37.516 INFO:tasks.rados.rados.0.smithi033.stdout:2615: done (1 left) 2024-09-21T09:59:37.516 INFO:tasks.rados.rados.0.smithi033.stdout:2616: done (0 left) 2024-09-21T09:59:37.516 INFO:tasks.rados.rados.0.smithi033.stdout:2617: delete oid 5 current snap is 243 2024-09-21T09:59:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:37 smithi105 ceph-mon[86649]: pgmap v450: 121 pgs: 121 active+clean; 328 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 438 KiB/s wr, 9 op/s 2024-09-21T09:59:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:37 smithi105 ceph-mon[86649]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:59:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:37 smithi105 ceph-mon[86649]: osd.7 marked itself down and dead 2024-09-21T09:59:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:37 smithi033 ceph-mon[98546]: pgmap v450: 121 pgs: 121 active+clean; 328 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 438 KiB/s wr, 9 op/s 2024-09-21T09:59:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:37 smithi033 ceph-mon[98546]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:59:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:37 smithi033 ceph-mon[98546]: osd.7 marked itself down and dead 2024-09-21T09:59:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:37 smithi033 ceph-mon[96568]: pgmap v450: 121 pgs: 121 active+clean; 328 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 438 KiB/s wr, 9 op/s 2024-09-21T09:59:37.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:37 smithi033 ceph-mon[96568]: osdmap e547: 8 total, 8 up, 8 in 2024-09-21T09:59:37.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:37 smithi033 ceph-mon[96568]: osd.7 marked itself down and dead 2024-09-21T09:59:38.351 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:38 smithi105 podman[106076]: 2024-09-21 09:59:38.037284856 +0000 UTC m=+1.444045144 container died 742c27449369130e6b7d28412a0960c799954e30909cd836f6caebd095ebf24b (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T09:59:38.524 INFO:tasks.rados.rados.0.smithi033.stdout:2617: done (0 left) 2024-09-21T09:59:38.525 INFO:tasks.rados.rados.0.smithi033.stdout:2618: delete oid 6 current snap is 243 2024-09-21T09:59:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2618: done (0 left) 2024-09-21T09:59:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2619: read oid 20 snap 214 2024-09-21T09:59:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2619: expect (ObjNum 91297376 snap 21955 seq_num 91092176) 2024-09-21T09:59:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2620: setattr oid 26 current snap is 243 2024-09-21T09:59:38.528 INFO:tasks.rados.rados.0.smithi033.stdout:2619: expect (ObjNum 91297376 snap 21955 seq_num 91092176) 2024-09-21T09:59:38.529 INFO:tasks.rados.rados.0.smithi033.stdout:2619: done (1 left) 2024-09-21T09:59:38.529 INFO:tasks.rados.rados.0.smithi033.stdout:2621: rollback oid 17 current snap is 243 2024-09-21T09:59:38.530 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 240 2024-09-21T09:59:38.530 INFO:tasks.rados.rados.0.smithi033.stdout:2622: snap_remove snap 219 2024-09-21T09:59:38.767 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:38 smithi105 podman[106076]: 2024-09-21 09:59:38.530797172 +0000 UTC m=+1.937557449 container cleanup 742c27449369130e6b7d28412a0960c799954e30909cd836f6caebd095ebf24b (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=quincy-fbd1a6a, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-21T09:59:38.767 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:38 smithi105 bash[106076]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7 2024-09-21T09:59:38.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:38 smithi105 ceph-mon[86649]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:59:38.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:38 smithi105 ceph-mon[86649]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T09:59:38.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:38 smithi105 ceph-mon[86649]: osdmap e548: 8 total, 7 up, 8 in 2024-09-21T09:59:38.768 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:38 smithi105 ceph-mon[86649]: osdmap e549: 8 total, 7 up, 8 in 2024-09-21T09:59:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:38 smithi033 ceph-mon[98546]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:59:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:38 smithi033 ceph-mon[98546]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T09:59:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:38 smithi033 ceph-mon[98546]: osdmap e548: 8 total, 7 up, 8 in 2024-09-21T09:59:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:38 smithi033 ceph-mon[98546]: osdmap e549: 8 total, 7 up, 8 in 2024-09-21T09:59:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:38 smithi033 ceph-mon[96568]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-21T09:59:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:38 smithi033 ceph-mon[96568]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T09:59:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:38 smithi033 ceph-mon[96568]: osdmap e548: 8 total, 7 up, 8 in 2024-09-21T09:59:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:38 smithi033 ceph-mon[96568]: osdmap e549: 8 total, 7 up, 8 in 2024-09-21T09:59:39.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:38 smithi105 podman[106095]: 2024-09-21 09:59:38.765959794 +0000 UTC m=+0.731759931 container remove 742c27449369130e6b7d28412a0960c799954e30909cd836f6caebd095ebf24b (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T09:59:39.359 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:39 smithi105 podman[106190]: 2024-09-21 09:59:39.005510606 +0000 UTC m=+0.015734792 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:59:39.359 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:39 smithi105 podman[106190]: 2024-09-21 09:59:39.117034689 +0000 UTC m=+0.127258920 container create 70c17929b3c2d2735a1623e9a75a6e6bed5b5ef77cd4e431f1fce9de5b11cbf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-21T09:59:39.359 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:39 smithi105 podman[106190]: 2024-09-21 09:59:39.352657593 +0000 UTC m=+0.362881779 container init 70c17929b3c2d2735a1623e9a75a6e6bed5b5ef77cd4e431f1fce9de5b11cbf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-21T09:59:39.544 INFO:tasks.rados.rados.0.smithi033.stdout:2622: done (2 left) 2024-09-21T09:59:39.544 INFO:tasks.rados.rados.0.smithi033.stdout:2623: delete oid 48 current snap is 243 2024-09-21T09:59:39.544 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1491 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T09:59:39.544 INFO:tasks.rados.rados.0.smithi033.stdout:2621: finishing rollback tid 1 to smithi03385152-17 2024-09-21T09:59:39.544 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:59:39.552 INFO:tasks.rados.rados.0.smithi033.stdout:2620: done (2 left) 2024-09-21T09:59:39.552 INFO:tasks.rados.rados.0.smithi033.stdout:2621: done (1 left) 2024-09-21T09:59:39.552 INFO:tasks.rados.rados.0.smithi033.stdout:2623: done (0 left) 2024-09-21T09:59:39.552 INFO:tasks.rados.rados.0.smithi033.stdout:2624: setattr oid 14 current snap is 243 2024-09-21T09:59:39.553 INFO:tasks.rados.rados.0.smithi033.stdout:2625: snap_remove snap 231 2024-09-21T09:59:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:39 smithi105 ceph-mon[86649]: pgmap v453: 121 pgs: 5 peering, 19 stale+active+clean, 97 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 658 KiB/s wr, 10 op/s 2024-09-21T09:59:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:39 smithi105 ceph-mon[86649]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:59:39.780 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:39 smithi105 podman[106190]: 2024-09-21 09:59:39.357465384 +0000 UTC m=+0.367689571 container start 70c17929b3c2d2735a1623e9a75a6e6bed5b5ef77cd4e431f1fce9de5b11cbf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T09:59:39.780 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:39 smithi105 podman[106190]: 2024-09-21 09:59:39.460879007 +0000 UTC m=+0.471103200 container attach 70c17929b3c2d2735a1623e9a75a6e6bed5b5ef77cd4e431f1fce9de5b11cbf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-21T09:59:39.780 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:39 smithi105 podman[106190]: 2024-09-21 09:59:39.552554582 +0000 UTC m=+0.562778772 container died 70c17929b3c2d2735a1623e9a75a6e6bed5b5ef77cd4e431f1fce9de5b11cbf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-21T09:59:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:39 smithi033 ceph-mon[98546]: pgmap v453: 121 pgs: 5 peering, 19 stale+active+clean, 97 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 658 KiB/s wr, 10 op/s 2024-09-21T09:59:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:39 smithi033 ceph-mon[98546]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:59:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:39 smithi033 ceph-mon[96568]: pgmap v453: 121 pgs: 5 peering, 19 stale+active+clean, 97 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 658 KiB/s wr, 10 op/s 2024-09-21T09:59:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:39 smithi033 ceph-mon[96568]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-21T09:59:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:59:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:59:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:40.530 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 podman[106233]: 2024-09-21 09:59:40.16446887 +0000 UTC m=+0.604638287 container remove 70c17929b3c2d2735a1623e9a75a6e6bed5b5ef77cd4e431f1fce9de5b11cbf1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T09:59:40.530 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.7.service: Deactivated successfully. 2024-09-21T09:59:40.530 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 systemd[1]: Stopped Ceph osd.7 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:59:40.531 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.7.service: Consumed 16.988s CPU time. 2024-09-21T09:59:40.531 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 systemd[1]: Starting Ceph osd.7 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:59:40.547 INFO:tasks.rados.rados.0.smithi033.stdout:2625: done (1 left) 2024-09-21T09:59:40.547 INFO:tasks.rados.rados.0.smithi033.stdout:2626: rollback oid 37 current snap is 243 2024-09-21T09:59:40.547 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 233 2024-09-21T09:59:40.547 INFO:tasks.rados.rados.0.smithi033.stdout:2627: snap_remove snap 220 2024-09-21T09:59:40.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:40 smithi105 ceph-mon[86649]: osdmap e550: 8 total, 7 up, 8 in 2024-09-21T09:59:40.904 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 podman[106337]: 2024-09-21 09:59:40.502587985 +0000 UTC m=+0.015784495 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:59:40.904 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 podman[106337]: 2024-09-21 09:59:40.641072196 +0000 UTC m=+0.154268690 container create 614826db2b9c7bbec9fc9a9fc50395d18ba2b35782f7666b478610b382f04961 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T09:59:40.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:40 smithi033 ceph-mon[96568]: osdmap e550: 8 total, 7 up, 8 in 2024-09-21T09:59:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:40 smithi033 ceph-mon[98546]: osdmap e550: 8 total, 7 up, 8 in 2024-09-21T09:59:41.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 podman[106337]: 2024-09-21 09:59:40.902005525 +0000 UTC m=+0.415202023 container init 614826db2b9c7bbec9fc9a9fc50395d18ba2b35782f7666b478610b382f04961 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, ceph=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-09-21T09:59:41.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 podman[106337]: 2024-09-21 09:59:40.907398148 +0000 UTC m=+0.420594644 container start 614826db2b9c7bbec9fc9a9fc50395d18ba2b35782f7666b478610b382f04961 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, maintainer=Guillaume Abrioux ) 2024-09-21T09:59:41.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:40 smithi105 podman[106337]: 2024-09-21 09:59:40.985076822 +0000 UTC m=+0.498273329 container attach 614826db2b9c7bbec9fc9a9fc50395d18ba2b35782f7666b478610b382f04961 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-21T09:59:41.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:59:41.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:59:41.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:59:41.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-21T09:59:41.547 INFO:tasks.rados.rados.0.smithi033.stdout:2627: done (2 left) 2024-09-21T09:59:41.547 INFO:tasks.rados.rados.0.smithi033.stdout:2628: copy_from oid 2 from oid 50 current snap is 243 2024-09-21T09:59:41.547 INFO:tasks.rados.rados.0.smithi033.stdout:2629: read oid 36 snap -1 2024-09-21T09:59:41.547 INFO:tasks.rados.rados.0.smithi033.stdout:2629: expect deleted 2024-09-21T09:59:41.547 INFO:tasks.rados.rados.0.smithi033.stdout:2630: snap_create 2024-09-21T09:59:41.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1493 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T09:59:41.547 INFO:tasks.rados.rados.0.smithi033.stdout:2626: finishing rollback tid 0 to smithi03385152-37 2024-09-21T09:59:41.547 INFO:tasks.rados.rados.0.smithi033.stdout:2626: finishing rollback tid 1 to smithi03385152-37 2024-09-21T09:59:41.548 INFO:tasks.rados.rados.0.smithi033.stdout:2626: finishing rollback tid 2 to smithi03385152-37 2024-09-21T09:59:41.548 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1052 (ObjNum 731 snap 210 seq_num 731) dirty exists 2024-09-21T09:59:41.553 INFO:tasks.rados.rados.0.smithi033.stdout:2628: finishing copy_from to smithi03385152-2 2024-09-21T09:59:41.553 INFO:tasks.rados.rados.0.smithi033.stdout:2628: got expected ENOENT (src dne) 2024-09-21T09:59:41.554 INFO:tasks.rados.rados.0.smithi033.stdout:2628: finishing copy_from racing read to smithi03385152-2 2024-09-21T09:59:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-mon[86649]: pgmap v456: 121 pgs: 11 active+undersized, 5 peering, 11 stale+active+clean, 6 active+undersized+degraded, 88 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 662 KiB/s wr, 9 op/s; 41/624 objects degraded (6.571%) 2024-09-21T09:59:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-mon[86649]: Health check failed: Degraded data redundancy: 41/624 objects degraded (6.571%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:41.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-mon[86649]: osdmap e551: 8 total, 7 up, 8 in 2024-09-21T09:59:41.781 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T09:59:41.782 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T09:59:41.782 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:59:41.782 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-21T09:59:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:59:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:59:41] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.51.0" 2024-09-21T09:59:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:41 smithi033 ceph-mon[98546]: pgmap v456: 121 pgs: 11 active+undersized, 5 peering, 11 stale+active+clean, 6 active+undersized+degraded, 88 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 662 KiB/s wr, 9 op/s; 41/624 objects degraded (6.571%) 2024-09-21T09:59:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:41 smithi033 ceph-mon[98546]: Health check failed: Degraded data redundancy: 41/624 objects degraded (6.571%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:41 smithi033 ceph-mon[98546]: osdmap e551: 8 total, 7 up, 8 in 2024-09-21T09:59:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:41 smithi033 ceph-mon[96568]: pgmap v456: 121 pgs: 11 active+undersized, 5 peering, 11 stale+active+clean, 6 active+undersized+degraded, 88 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 662 KiB/s wr, 9 op/s; 41/624 objects degraded (6.571%) 2024-09-21T09:59:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:41 smithi033 ceph-mon[96568]: Health check failed: Degraded data redundancy: 41/624 objects degraded (6.571%), 6 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:41 smithi033 ceph-mon[96568]: osdmap e551: 8 total, 7 up, 8 in 2024-09-21T09:59:42.157 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:59:42.157 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-21T09:59:42.157 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:59:42.158 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-21T09:59:42.158 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T09:59:42.158 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-09-21T09:59:42.158 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T09:59:42.158 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-09-21T09:59:42.158 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate[106352]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-21T09:59:42.158 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 bash[106337]: --> ceph-volume raw activate successful for osd ID: 7 2024-09-21T09:59:42.158 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:41 smithi105 podman[106337]: 2024-09-21 09:59:41.831165044 +0000 UTC m=+1.344361540 container died 614826db2b9c7bbec9fc9a9fc50395d18ba2b35782f7666b478610b382f04961 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True) 2024-09-21T09:59:42.547 INFO:tasks.rados.rados.0.smithi033.stdout:2624: done (4 left) 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2626: done (3 left) 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2628: done (2 left) 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2629: done (1 left) 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2630: done (0 left) 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2631: rmattr oid 7 current snap is 244 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2632: read oid 45 snap -1 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2632: expect deleted 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2633: read oid 15 snap -1 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2633: expect (ObjNum 684 snap 188 seq_num 684) 2024-09-21T09:59:42.548 INFO:tasks.rados.rados.0.smithi033.stdout:2634: snap_remove snap 214 2024-09-21T09:59:42.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:42 smithi105 ceph-mon[86649]: osdmap e552: 8 total, 7 up, 8 in 2024-09-21T09:59:42.767 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:42 smithi105 ceph-mon[86649]: osdmap e553: 8 total, 7 up, 8 in 2024-09-21T09:59:42.767 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:42 smithi105 podman[106497]: 2024-09-21 09:59:42.460621753 +0000 UTC m=+0.623170473 container remove 614826db2b9c7bbec9fc9a9fc50395d18ba2b35782f7666b478610b382f04961 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:59:42.767 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:42 smithi105 podman[106563]: 2024-09-21 09:59:42.665076822 +0000 UTC m=+0.112053507 container create ea10ccd8e9ee9580ab0c729f3addb87e59e1f7e048bff1bd4567b9602cbac03d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:59:42.767 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:42 smithi105 podman[106563]: 2024-09-21 09:59:42.569897209 +0000 UTC m=+0.016873829 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:59:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:42 smithi033 ceph-mon[98546]: osdmap e552: 8 total, 7 up, 8 in 2024-09-21T09:59:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:42 smithi033 ceph-mon[98546]: osdmap e553: 8 total, 7 up, 8 in 2024-09-21T09:59:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:42 smithi033 ceph-mon[96568]: osdmap e552: 8 total, 7 up, 8 in 2024-09-21T09:59:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:42 smithi033 ceph-mon[96568]: osdmap e553: 8 total, 7 up, 8 in 2024-09-21T09:59:43.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:42 smithi105 podman[106563]: 2024-09-21 09:59:42.859135135 +0000 UTC m=+0.306111763 container init ea10ccd8e9ee9580ab0c729f3addb87e59e1f7e048bff1bd4567b9602cbac03d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-21T09:59:43.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:42 smithi105 podman[106563]: 2024-09-21 09:59:42.864203769 +0000 UTC m=+0.311180391 container start ea10ccd8e9ee9580ab0c729f3addb87e59e1f7e048bff1bd4567b9602cbac03d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T09:59:43.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:42 smithi105 ceph-osd[106602]: -- 172.21.15.105:0/1421566891 <== mon.1 v2:172.21.15.105:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5646b205fa40 con 0x5646b203dc00 2024-09-21T09:59:43.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:42 smithi105 bash[106563]: ea10ccd8e9ee9580ab0c729f3addb87e59e1f7e048bff1bd4567b9602cbac03d 2024-09-21T09:59:43.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:42 smithi105 systemd[1]: Started Ceph osd.7 for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:59:43.508 INFO:tasks.rados.rados.0.smithi033.stdout:2634: done (3 left) 2024-09-21T09:59:43.508 INFO:tasks.rados.rados.0.smithi033.stdout:2635: delete oid 16 current snap is 244 2024-09-21T09:59:43.508 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 1853 (ObjNum 724 snap 205 seq_num 724) dirty exists 2024-09-21T09:59:43.508 INFO:tasks.rados.rados.0.smithi033.stdout:2633: expect (ObjNum 684 snap 188 seq_num 684) 2024-09-21T09:59:43.609 INFO:tasks.rados.rados.0.smithi033.stdout:2631: done (3 left) 2024-09-21T09:59:43.609 INFO:tasks.rados.rados.0.smithi033.stdout:2632: done (2 left) 2024-09-21T09:59:43.609 INFO:tasks.rados.rados.0.smithi033.stdout:2633: done (1 left) 2024-09-21T09:59:43.609 INFO:tasks.rados.rados.0.smithi033.stdout:2635: done (0 left) 2024-09-21T09:59:43.609 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 244 2024-09-21T09:59:43.609 INFO:tasks.rados.rados.0.smithi033.stdout:2636: seq_num 832 ranges {13508608=737280,14245888=786432,15032320=417792} 2024-09-21T09:59:43.686 INFO:tasks.rados.rados.0.smithi033.stdout:2636: writing smithi03385152-29 from 13508608 to 14245888 tid 1 2024-09-21T09:59:43.690 INFO:tasks.rados.rados.0.smithi033.stdout:2636: writing smithi03385152-29 from 14245888 to 15032320 tid 2 2024-09-21T09:59:43.693 INFO:tasks.rados.rados.0.smithi033.stdout:2636: writing smithi03385152-29 from 15032320 to 15450112 tid 3 2024-09-21T09:59:43.693 INFO:tasks.rados.rados.0.smithi033.stdout:2637: copy_from oid 21 from oid 20 current snap is 244 2024-09-21T09:59:43.693 INFO:tasks.rados.rados.0.smithi033.stdout:2638: snap_create 2024-09-21T09:59:43.694 INFO:tasks.rados.rados.0.smithi033.stdout:2636: finishing write tid 1 to smithi03385152-29 2024-09-21T09:59:43.696 INFO:tasks.rados.rados.0.smithi033.stdout:2636: finishing write tid 2 to smithi03385152-29 2024-09-21T09:59:43.698 INFO:tasks.rados.rados.0.smithi033.stdout:2636: finishing write tid 3 to smithi03385152-29 2024-09-21T09:59:43.698 INFO:tasks.rados.rados.0.smithi033.stdout:2636: finishing write tid 4 to smithi03385152-29 2024-09-21T09:59:43.699 INFO:tasks.rados.rados.0.smithi033.stdout:2636: finishing write tid 5 to smithi03385152-29 2024-09-21T09:59:43.699 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1872 (ObjNum 832 snap 244 seq_num 832) dirty exists 2024-09-21T09:59:43.699 INFO:tasks.rados.rados.0.smithi033.stdout:2636: left oid 29 (ObjNum 832 snap 244 seq_num 832) 2024-09-21T09:59:43.701 INFO:tasks.rados.rados.0.smithi033.stdout:2637: finishing copy_from racing read to smithi03385152-21 2024-09-21T09:59:43.703 INFO:tasks.rados.rados.0.smithi033.stdout:2637: finishing copy_from to smithi03385152-21 2024-09-21T09:59:43.703 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1496 (ObjNum 807 snap 234 seq_num 807) dirty exists 2024-09-21T09:59:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[98546]: pgmap v459: 121 pgs: 32 active+undersized, 5 peering, 13 active+undersized+degraded, 71 active+clean; 311 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s wr, 12 op/s; 61/630 objects degraded (9.683%) 2024-09-21T09:59:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[98546]: osdmap e554: 8 total, 7 up, 8 in 2024-09-21T09:59:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[96568]: pgmap v459: 121 pgs: 32 active+undersized, 5 peering, 13 active+undersized+degraded, 71 active+clean; 311 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s wr, 12 op/s; 61/630 objects degraded (9.683%) 2024-09-21T09:59:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:43 smithi033 ceph-mon[96568]: osdmap e554: 8 total, 7 up, 8 in 2024-09-21T09:59:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:43 smithi105 ceph-mon[86649]: pgmap v459: 121 pgs: 32 active+undersized, 5 peering, 13 active+undersized+degraded, 71 active+clean; 311 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 896 KiB/s wr, 12 op/s; 61/630 objects degraded (9.683%) 2024-09-21T09:59:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:43 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:43 smithi105 ceph-mon[86649]: osdmap e554: 8 total, 7 up, 8 in 2024-09-21T09:59:44.566 INFO:tasks.rados.rados.0.smithi033.stdout:2636: done (2 left) 2024-09-21T09:59:44.566 INFO:tasks.rados.rados.0.smithi033.stdout:2637: done (1 left) 2024-09-21T09:59:44.566 INFO:tasks.rados.rados.0.smithi033.stdout:2638: done (0 left) 2024-09-21T09:59:44.566 INFO:tasks.rados.rados.0.smithi033.stdout:2639: setattr oid 10 current snap is 245 2024-09-21T09:59:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:2640: copy_from oid 36 from oid 20 current snap is 245 2024-09-21T09:59:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:2641: copy_from oid 12 from oid 48 current snap is 245 2024-09-21T09:59:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:2642: snap_create 2024-09-21T09:59:44.571 INFO:tasks.rados.rados.0.smithi033.stdout:2641: finishing copy_from to smithi03385152-12 2024-09-21T09:59:44.571 INFO:tasks.rados.rados.0.smithi033.stdout:2641: got expected ENOENT (src dne) 2024-09-21T09:59:44.571 INFO:tasks.rados.rados.0.smithi033.stdout:2641: finishing copy_from racing read to smithi03385152-12 2024-09-21T09:59:44.574 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1057 (ObjNum 818 snap 238 seq_num 818) dirty exists 2024-09-21T09:59:44.580 INFO:tasks.rados.rados.0.smithi033.stdout:2640: finishing copy_from racing read to smithi03385152-36 2024-09-21T09:59:44.581 INFO:tasks.rados.rados.0.smithi033.stdout:2640: finishing copy_from to smithi03385152-36 2024-09-21T09:59:44.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1496 (ObjNum 807 snap 234 seq_num 807) dirty exists 2024-09-21T09:59:44.801 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:44 smithi105 ceph-mon[86649]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:59:44.801 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:44 smithi105 ceph-mon[86649]: osdmap e555: 8 total, 7 up, 8 in 2024-09-21T09:59:44.801 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T09:59:44.755+0000 7fc1aed55640 -1 Falling back to public interface 2024-09-21T09:59:44.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:44 smithi033 ceph-mon[98546]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:59:44.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:44 smithi033 ceph-mon[98546]: osdmap e555: 8 total, 7 up, 8 in 2024-09-21T09:59:44.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:44 smithi033 ceph-mon[96568]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-21T09:59:44.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:44 smithi033 ceph-mon[96568]: osdmap e555: 8 total, 7 up, 8 in 2024-09-21T09:59:45.569 INFO:tasks.rados.rados.0.smithi033.stdout:2639: done (3 left) 2024-09-21T09:59:45.569 INFO:tasks.rados.rados.0.smithi033.stdout:2640: done (2 left) 2024-09-21T09:59:45.569 INFO:tasks.rados.rados.0.smithi033.stdout:2641: done (1 left) 2024-09-21T09:59:45.569 INFO:tasks.rados.rados.0.smithi033.stdout:2642: done (0 left) 2024-09-21T09:59:45.569 INFO:tasks.rados.rados.0.smithi033.stdout:2643: rmattr oid 47 current snap is 246 2024-09-21T09:59:45.569 INFO:tasks.rados.rados.0.smithi033.stdout:2643: done (0 left) 2024-09-21T09:59:45.569 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 246 2024-09-21T09:59:45.569 INFO:tasks.rados.rados.0.smithi033.stdout:2644: seq_num 833 ranges {0=761856,761856=671744,1433600=532480} 2024-09-21T09:59:45.572 INFO:tasks.rados.rados.0.smithi033.stdout:2644: writing smithi03385152-50 from 0 to 761856 tid 1 2024-09-21T09:59:45.576 INFO:tasks.rados.rados.0.smithi033.stdout:2644: writing smithi03385152-50 from 761856 to 1433600 tid 2 2024-09-21T09:59:45.580 INFO:tasks.rados.rados.0.smithi033.stdout:2644: writing smithi03385152-50 from 1433600 to 1966080 tid 3 2024-09-21T09:59:45.580 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 246 2024-09-21T09:59:45.580 INFO:tasks.rados.rados.0.smithi033.stdout:2644: finishing write tid 1 to smithi03385152-50 2024-09-21T09:59:45.580 INFO:tasks.rados.rados.0.smithi033.stdout:2645: seq_num 834 ranges {245760=778240,1024000=557056,1581056=548864,2129920=98304} 2024-09-21T09:59:45.586 INFO:tasks.rados.rados.0.smithi033.stdout:2645: writing smithi03385152-37 from 245760 to 1024000 tid 1 2024-09-21T09:59:45.589 INFO:tasks.rados.rados.0.smithi033.stdout:2645: writing smithi03385152-37 from 1024000 to 1581056 tid 2 2024-09-21T09:59:45.593 INFO:tasks.rados.rados.0.smithi033.stdout:2645: writing smithi03385152-37 from 1581056 to 2129920 tid 3 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:2645: writing smithi03385152-37 from 2129920 to 2228224 tid 4 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 246 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:2644: finishing write tid 2 to smithi03385152-50 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:2644: finishing write tid 3 to smithi03385152-50 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:2644: finishing write tid 4 to smithi03385152-50 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:2644: finishing write tid 5 to smithi03385152-50 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 939 (ObjNum 833 snap 246 seq_num 833) dirty exists 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:2644: left oid 50 (ObjNum 833 snap 246 seq_num 833) 2024-09-21T09:59:45.594 INFO:tasks.rados.rados.0.smithi033.stdout:2646: seq_num 835 ranges {0=794624,794624=442368,1236992=712704,1949696=49152} 2024-09-21T09:59:45.599 INFO:tasks.rados.rados.0.smithi033.stdout:2646: writing smithi03385152-47 from 0 to 794624 tid 1 2024-09-21T09:59:45.601 INFO:tasks.rados.rados.0.smithi033.stdout:2646: writing smithi03385152-47 from 794624 to 1236992 tid 2 2024-09-21T09:59:45.606 INFO:tasks.rados.rados.0.smithi033.stdout:2646: writing smithi03385152-47 from 1236992 to 1949696 tid 3 2024-09-21T09:59:45.606 INFO:tasks.rados.rados.0.smithi033.stdout:2646: writing smithi03385152-47 from 1949696 to 1998848 tid 4 2024-09-21T09:59:45.606 INFO:tasks.rados.rados.0.smithi033.stdout:2644: done (2 left) 2024-09-21T09:59:45.606 INFO:tasks.rados.rados.0.smithi033.stdout:2647: delete oid 30 current snap is 246 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2645: finishing write tid 1 to smithi03385152-37 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2645: finishing write tid 2 to smithi03385152-37 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2645: finishing write tid 3 to smithi03385152-37 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2645: finishing write tid 4 to smithi03385152-37 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2645: finishing write tid 5 to smithi03385152-37 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2645: finishing write tid 6 to smithi03385152-37 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1057 (ObjNum 834 snap 246 seq_num 834) dirty exists 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2645: left oid 37 (ObjNum 834 snap 246 seq_num 834) 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2646: finishing write tid 1 to smithi03385152-47 2024-09-21T09:59:45.607 INFO:tasks.rados.rados.0.smithi033.stdout:2646: finishing write tid 2 to smithi03385152-47 2024-09-21T09:59:45.609 INFO:tasks.rados.rados.0.smithi033.stdout:2645: done (2 left) 2024-09-21T09:59:45.609 INFO:tasks.rados.rados.0.smithi033.stdout:2647: done (1 left) 2024-09-21T09:59:45.609 INFO:tasks.rados.rados.0.smithi033.stdout:2648: snap_create 2024-09-21T09:59:45.610 INFO:tasks.rados.rados.0.smithi033.stdout:2646: finishing write tid 3 to smithi03385152-47 2024-09-21T09:59:45.610 INFO:tasks.rados.rados.0.smithi033.stdout:2646: finishing write tid 4 to smithi03385152-47 2024-09-21T09:59:45.610 INFO:tasks.rados.rados.0.smithi033.stdout:2646: finishing write tid 5 to smithi03385152-47 2024-09-21T09:59:45.612 INFO:tasks.rados.rados.0.smithi033.stdout:2646: finishing write tid 6 to smithi03385152-47 2024-09-21T09:59:45.612 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1483 (ObjNum 835 snap 246 seq_num 835) dirty exists 2024-09-21T09:59:45.612 INFO:tasks.rados.rados.0.smithi033.stdout:2646: left oid 47 (ObjNum 835 snap 246 seq_num 835) 2024-09-21T09:59:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:45 smithi033 ceph-mon[98546]: pgmap v462: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 454 KiB/s wr, 6 op/s; 61/612 objects degraded (9.967%) 2024-09-21T09:59:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:45 smithi033 ceph-mon[98546]: osdmap e556: 8 total, 7 up, 8 in 2024-09-21T09:59:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:45 smithi033 ceph-mon[96568]: pgmap v462: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 454 KiB/s wr, 6 op/s; 61/612 objects degraded (9.967%) 2024-09-21T09:59:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:45 smithi033 ceph-mon[96568]: osdmap e556: 8 total, 7 up, 8 in 2024-09-21T09:59:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:45 smithi105 ceph-mon[86649]: pgmap v462: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 454 KiB/s wr, 6 op/s; 61/612 objects degraded (9.967%) 2024-09-21T09:59:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:45 smithi105 ceph-mon[86649]: osdmap e556: 8 total, 7 up, 8 in 2024-09-21T09:59:46.570 INFO:tasks.rados.rados.0.smithi033.stdout:2646: done (1 left) 2024-09-21T09:59:46.571 INFO:tasks.rados.rados.0.smithi033.stdout:2648: done (0 left) 2024-09-21T09:59:46.571 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 247 2024-09-21T09:59:46.571 INFO:tasks.rados.rados.0.smithi033.stdout:2649: seq_num 837 ranges {0=425984,425984=655360,1081344=729088,1810432=221184} 2024-09-21T09:59:46.572 INFO:tasks.rados.rados.0.smithi033.stdout:2649: writing smithi03385152-48 from 0 to 425984 tid 1 2024-09-21T09:59:46.576 INFO:tasks.rados.rados.0.smithi033.stdout:2649: writing smithi03385152-48 from 425984 to 1081344 tid 2 2024-09-21T09:59:46.580 INFO:tasks.rados.rados.0.smithi033.stdout:2649: writing smithi03385152-48 from 1081344 to 1810432 tid 3 2024-09-21T09:59:46.582 INFO:tasks.rados.rados.0.smithi033.stdout:2649: writing smithi03385152-48 from 1810432 to 2031616 tid 4 2024-09-21T09:59:46.582 INFO:tasks.rados.rados.0.smithi033.stdout:2650: delete oid 9 current snap is 247 2024-09-21T09:59:46.582 INFO:tasks.rados.rados.0.smithi033.stdout:2649: finishing write tid 1 to smithi03385152-48 2024-09-21T09:59:46.582 INFO:tasks.rados.rados.0.smithi033.stdout:2649: finishing write tid 2 to smithi03385152-48 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2650: done (1 left) 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2651: read oid 25 snap -1 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2651: expect (ObjNum 787 snap 230 seq_num 787) 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2652: rmattr oid 34 current snap is 247 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2653: read oid 29 snap 245 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2653: expect (ObjNum 832 snap 244 seq_num 832) 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2654: snap_create 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2649: finishing write tid 3 to smithi03385152-48 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2649: finishing write tid 4 to smithi03385152-48 2024-09-21T09:59:46.587 INFO:tasks.rados.rados.0.smithi033.stdout:2649: finishing write tid 5 to smithi03385152-48 2024-09-21T09:59:46.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1514 (ObjNum 797 snap 233 seq_num 797) dirty exists 2024-09-21T09:59:46.597 INFO:tasks.rados.rados.0.smithi033.stdout:2649: finishing write tid 6 to smithi03385152-48 2024-09-21T09:59:46.597 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 1512 (ObjNum 837 snap 247 seq_num 837) dirty exists 2024-09-21T09:59:46.597 INFO:tasks.rados.rados.0.smithi033.stdout:2649: left oid 48 (ObjNum 837 snap 247 seq_num 837) 2024-09-21T09:59:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:2651: expect (ObjNum 787 snap 230 seq_num 787) 2024-09-21T09:59:46.830 INFO:tasks.rados.rados.0.smithi033.stdout:2653: expect (ObjNum 832 snap 244 seq_num 832) 2024-09-21T09:59:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:46 smithi105 ceph-mon[86649]: osdmap e557: 8 total, 7 up, 8 in 2024-09-21T09:59:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:46 smithi033 ceph-mon[98546]: osdmap e557: 8 total, 7 up, 8 in 2024-09-21T09:59:47.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:47.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T09:59:47.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:46 smithi033 ceph-mon[96568]: osdmap e557: 8 total, 7 up, 8 in 2024-09-21T09:59:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:47 smithi105 ceph-mon[86649]: pgmap v465: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 295 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.7 MiB/s wr, 6 op/s; 61/615 objects degraded (9.919%) 2024-09-21T09:59:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:47 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 61/615 objects degraded (9.919%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:47 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:47 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:47 smithi105 ceph-mon[86649]: osdmap e558: 8 total, 7 up, 8 in 2024-09-21T09:59:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[98546]: pgmap v465: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 295 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.7 MiB/s wr, 6 op/s; 61/615 objects degraded (9.919%) 2024-09-21T09:59:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 61/615 objects degraded (9.919%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:48.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:48.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:48.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[98546]: osdmap e558: 8 total, 7 up, 8 in 2024-09-21T09:59:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[96568]: pgmap v465: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 295 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 1.7 MiB/s wr, 6 op/s; 61/615 objects degraded (9.919%) 2024-09-21T09:59:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 61/615 objects degraded (9.919%), 13 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:47 smithi033 ceph-mon[96568]: osdmap e558: 8 total, 7 up, 8 in 2024-09-21T09:59:48.317 INFO:tasks.rados.rados.0.smithi033.stdout:2649: done (4 left) 2024-09-21T09:59:48.317 INFO:tasks.rados.rados.0.smithi033.stdout:2651: done (3 left) 2024-09-21T09:59:48.317 INFO:tasks.rados.rados.0.smithi033.stdout:2652: done (2 left) 2024-09-21T09:59:48.317 INFO:tasks.rados.rados.0.smithi033.stdout:2653: done (1 left) 2024-09-21T09:59:48.317 INFO:tasks.rados.rados.0.smithi033.stdout:2654: done (0 left) 2024-09-21T09:59:48.317 INFO:tasks.rados.rados.0.smithi033.stdout:2655: snap_remove snap 241 2024-09-21T09:59:48.894 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T09:59:48.736+0000 7fc1aed55640 -1 osd.7 547 log_to_monitors true 2024-09-21T09:59:48.909 INFO:tasks.rados.rados.0.smithi033.stdout:2655: done (0 left) 2024-09-21T09:59:48.910 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 248 2024-09-21T09:59:48.910 INFO:tasks.rados.rados.0.smithi033.stdout:2656: seq_num 839 ranges {516096=458752,974848=425984,1400832=688128,2088960=491520} 2024-09-21T09:59:48.915 INFO:tasks.rados.rados.0.smithi033.stdout:2656: writing smithi03385152-2 from 516096 to 974848 tid 1 2024-09-21T09:59:48.918 INFO:tasks.rados.rados.0.smithi033.stdout:2656: writing smithi03385152-2 from 974848 to 1400832 tid 2 2024-09-21T09:59:48.922 INFO:tasks.rados.rados.0.smithi033.stdout:2656: writing smithi03385152-2 from 1400832 to 2088960 tid 3 2024-09-21T09:59:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:2656: writing smithi03385152-2 from 2088960 to 2580480 tid 4 2024-09-21T09:59:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:2657: read oid 25 snap -1 2024-09-21T09:59:48.926 INFO:tasks.rados.rados.0.smithi033.stdout:2657: expect (ObjNum 787 snap 230 seq_num 787) 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2658: read oid 32 snap 223 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2658: expect deleted 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2659: read oid 43 snap -1 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2659: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2660: copy_from oid 33 from oid 39 current snap is 248 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2661: read oid 8 snap 242 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2661: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2662: copy_from oid 5 from oid 4 current snap is 248 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 248 2024-09-21T09:59:48.927 INFO:tasks.rados.rados.0.smithi033.stdout:2663: seq_num 840 ranges {1523712=475136,1998848=712704,2711552=499712,3211264=393216} 2024-09-21T09:59:48.938 INFO:tasks.rados.rados.0.smithi033.stdout:2663: writing smithi03385152-36 from 1523712 to 1998848 tid 1 2024-09-21T09:59:48.943 INFO:tasks.rados.rados.0.smithi033.stdout:2663: writing smithi03385152-36 from 1998848 to 2711552 tid 2 2024-09-21T09:59:48.945 INFO:tasks.rados.rados.0.smithi033.stdout:2663: writing smithi03385152-36 from 2711552 to 3211264 tid 3 2024-09-21T09:59:48.948 INFO:tasks.rados.rados.0.smithi033.stdout:2663: writing smithi03385152-36 from 3211264 to 3604480 tid 4 2024-09-21T09:59:48.948 INFO:tasks.rados.rados.0.smithi033.stdout:2664: delete oid 13 current snap is 248 2024-09-21T09:59:48.948 INFO:tasks.rados.rados.0.smithi033.stdout:2659: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T09:59:48.949 INFO:tasks.rados.rados.0.smithi033.stdout:2661: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T09:59:48.960 INFO:tasks.rados.rados.0.smithi033.stdout:2657: expect (ObjNum 787 snap 230 seq_num 787) 2024-09-21T09:59:49.196 INFO:tasks.rados.rados.0.smithi033.stdout:2656: finishing write tid 1 to smithi03385152-2 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2656: finishing write tid 2 to smithi03385152-2 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2660: finishing copy_from racing read to smithi03385152-33 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2656: finishing write tid 3 to smithi03385152-2 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2656: finishing write tid 4 to smithi03385152-2 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2657: done (8 left) 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2658: done (7 left) 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2659: done (6 left) 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2661: done (5 left) 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2664: done (4 left) 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2665: copy_from oid 21 from oid 37 current snap is 248 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2656: finishing write tid 5 to smithi03385152-2 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2656: finishing write tid 6 to smithi03385152-2 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 944 (ObjNum 839 snap 248 seq_num 839) dirty exists 2024-09-21T09:59:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:2656: left oid 2 (ObjNum 839 snap 248 seq_num 839) 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2662: finishing copy_from racing read to smithi03385152-5 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2660: finishing copy_from to smithi03385152-33 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1021 (ObjNum 798 snap 233 seq_num 798) dirty exists 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2662: finishing copy_from to smithi03385152-5 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 1046 (ObjNum 746 snap 216 seq_num 746) dirty exists 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2663: finishing write tid 1 to smithi03385152-36 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2663: finishing write tid 2 to smithi03385152-36 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2663: finishing write tid 3 to smithi03385152-36 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2663: finishing write tid 4 to smithi03385152-36 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2663: finishing write tid 5 to smithi03385152-36 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2663: finishing write tid 6 to smithi03385152-36 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 1881 (ObjNum 840 snap 248 seq_num 840) dirty exists 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2663: left oid 36 (ObjNum 840 snap 248 seq_num 840) 2024-09-21T09:59:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:2656: done (4 left) 2024-09-21T09:59:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:2660: done (3 left) 2024-09-21T09:59:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:2662: done (2 left) 2024-09-21T09:59:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:2663: done (1 left) 2024-09-21T09:59:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:2666: snap_remove snap 216 2024-09-21T09:59:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:48 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:48 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:48 smithi033 ceph-mon[98546]: from='osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:59:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:48 smithi033 ceph-mon[98546]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:59:49.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:48 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:49.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:48 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:49.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:48 smithi033 ceph-mon[96568]: from='osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:59:49.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:48 smithi033 ceph-mon[96568]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:59:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:48 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:48 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:48 smithi105 ceph-mon[86649]: from='osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744]' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:59:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:48 smithi105 ceph-mon[86649]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]} : dispatch 2024-09-21T09:59:49.606 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 09:59:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T09:59:49.386+0000 7fc1a2af7640 -1 osd.7 547 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-21T09:59:49.912 INFO:tasks.rados.rados.0.smithi033.stdout:2665: finishing copy_from racing read to smithi03385152-21 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2665: finishing copy_from to smithi03385152-21 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1057 (ObjNum 834 snap 246 seq_num 834) dirty exists 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2665: done (1 left) 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2666: done (0 left) 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2667: read oid 2 snap 228 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2667: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2668: rollback oid 50 current snap is 248 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 50 to 239 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2669: read oid 5 snap -1 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2669: expect (ObjNum 746 snap 216 seq_num 746) 2024-09-21T09:59:49.913 INFO:tasks.rados.rados.0.smithi033.stdout:2670: read oid 3 snap 244 2024-09-21T09:59:49.914 INFO:tasks.rados.rados.0.smithi033.stdout:2670: expect deleted 2024-09-21T09:59:49.914 INFO:tasks.rados.rados.0.smithi033.stdout:2671: read oid 23 snap -1 2024-09-21T09:59:49.915 INFO:tasks.rados.rados.0.smithi033.stdout:2671: expect (ObjNum 827 snap 242 seq_num 827) 2024-09-21T09:59:49.915 INFO:tasks.rados.rados.0.smithi033.stdout:2672: rollback oid 17 current snap is 248 2024-09-21T09:59:49.915 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 204 2024-09-21T09:59:49.915 INFO:tasks.rados.rados.0.smithi033.stdout:2673: copy_from oid 36 from oid 1 current snap is 248 2024-09-21T09:59:49.915 INFO:tasks.rados.rados.0.smithi033.stdout:2674: delete oid 12 current snap is 248 2024-09-21T09:59:49.941 INFO:tasks.rados.rados.0.smithi033.stdout:2667: expect (ObjNum 747 snap 216 seq_num 747) 2024-09-21T09:59:49.961 INFO:tasks.rados.rados.0.smithi033.stdout:2668: finishing rollback tid 0 to smithi03385152-50 2024-09-21T09:59:49.962 INFO:tasks.rados.rados.0.smithi033.stdout:2668: finishing rollback tid 1 to smithi03385152-50 2024-09-21T09:59:49.962 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 946 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T09:59:49.973 INFO:tasks.rados.rados.0.smithi033.stdout:2671: expect (ObjNum 827 snap 242 seq_num 827) 2024-09-21T09:59:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:49 smithi105 ceph-mon[86649]: pgmap v468: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 295 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.7 MiB/s wr, 7 op/s; 61/615 objects degraded (9.919%) 2024-09-21T09:59:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:49 smithi105 ceph-mon[86649]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:59:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:49 smithi105 ceph-mon[86649]: from='osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:49 smithi105 ceph-mon[86649]: osdmap e559: 8 total, 7 up, 8 in 2024-09-21T09:59:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:49 smithi105 ceph-mon[86649]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:50.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:49 smithi105 ceph-mon[86649]: from='osd.7 ' entity='osd.7' 2024-09-21T09:59:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[96568]: pgmap v468: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 295 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.7 MiB/s wr, 7 op/s; 61/615 objects degraded (9.919%) 2024-09-21T09:59:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[96568]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:59:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[96568]: from='osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[96568]: osdmap e559: 8 total, 7 up, 8 in 2024-09-21T09:59:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[96568]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[96568]: from='osd.7 ' entity='osd.7' 2024-09-21T09:59:50.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[98546]: pgmap v468: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 295 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 1.7 MiB/s wr, 7 op/s; 61/615 objects degraded (9.919%) 2024-09-21T09:59:50.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[98546]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-21T09:59:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[98546]: from='osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744]' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[98546]: osdmap e559: 8 total, 7 up, 8 in 2024-09-21T09:59:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[98546]: from='osd.7 ' entity='osd.7' cmd={"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi105", "root=default"]} : dispatch 2024-09-21T09:59:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:49 smithi033 ceph-mon[98546]: from='osd.7 ' entity='osd.7' 2024-09-21T09:59:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 09:59:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T09:59:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T09:59:50.925 INFO:tasks.rados.rados.0.smithi033.stdout:2673: finishing copy_from to smithi03385152-36 2024-09-21T09:59:50.925 INFO:tasks.rados.rados.0.smithi033.stdout:2673: got expected ENOENT (src dne) 2024-09-21T09:59:50.925 INFO:tasks.rados.rados.0.smithi033.stdout:2673: finishing copy_from racing read to smithi03385152-36 2024-09-21T09:59:50.938 INFO:tasks.rados.rados.0.smithi033.stdout:2672: finishing rollback tid 1 to smithi03385152-17 2024-09-21T09:59:50.938 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 0 (ObjNum 593 snap 0 seq_num 114184296) dirty dne 2024-09-21T09:59:50.938 INFO:tasks.rados.rados.0.smithi033.stdout:2667: done (7 left) 2024-09-21T09:59:50.938 INFO:tasks.rados.rados.0.smithi033.stdout:2668: done (6 left) 2024-09-21T09:59:50.939 INFO:tasks.rados.rados.0.smithi033.stdout:2670: done (5 left) 2024-09-21T09:59:50.939 INFO:tasks.rados.rados.0.smithi033.stdout:2671: done (4 left) 2024-09-21T09:59:50.939 INFO:tasks.rados.rados.0.smithi033.stdout:2672: done (3 left) 2024-09-21T09:59:50.939 INFO:tasks.rados.rados.0.smithi033.stdout:2673: done (2 left) 2024-09-21T09:59:50.939 INFO:tasks.rados.rados.0.smithi033.stdout:2674: done (1 left) 2024-09-21T09:59:50.939 INFO:tasks.rados.rados.0.smithi033.stdout:2675: snap_remove snap 243 2024-09-21T09:59:51.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:50 smithi105 ceph-mon[86649]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:59:51.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:50 smithi105 ceph-mon[86649]: osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744] boot 2024-09-21T09:59:51.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:50 smithi105 ceph-mon[86649]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:59:51.011 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:59:51.012 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:50 smithi105 ceph-mon[86649]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:59:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[96568]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:59:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[96568]: osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744] boot 2024-09-21T09:59:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[96568]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:59:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:59:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[96568]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:59:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[98546]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-21T09:59:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[98546]: osd.7 [v2:172.21.15.105:6824/3020670744,v1:172.21.15.105:6825/3020670744] boot 2024-09-21T09:59:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[98546]: osdmap e560: 8 total, 8 up, 8 in 2024-09-21T09:59:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 7} : dispatch 2024-09-21T09:59:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:50 smithi033 ceph-mon[98546]: osdmap e561: 8 total, 8 up, 8 in 2024-09-21T09:59:51.924 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:09:59:51] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.51.0" 2024-09-21T09:59:51.935 INFO:tasks.rados.rados.0.smithi033.stdout:2675: done (1 left) 2024-09-21T09:59:51.935 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 248 2024-09-21T09:59:51.935 INFO:tasks.rados.rados.0.smithi033.stdout:2676: seq_num 843 ranges {1998848=524288,2523136=761856,3284992=671744,3956736=172032} 2024-09-21T09:59:51.950 INFO:tasks.rados.rados.0.smithi033.stdout:2676: writing smithi03385152-47 from 1998848 to 2523136 tid 1 2024-09-21T09:59:51.955 INFO:tasks.rados.rados.0.smithi033.stdout:2676: writing smithi03385152-47 from 2523136 to 3284992 tid 2 2024-09-21T09:59:51.959 INFO:tasks.rados.rados.0.smithi033.stdout:2676: writing smithi03385152-47 from 3284992 to 3956736 tid 3 2024-09-21T09:59:51.961 INFO:tasks.rados.rados.0.smithi033.stdout:2676: writing smithi03385152-47 from 3956736 to 4128768 tid 4 2024-09-21T09:59:51.961 INFO:tasks.rados.rados.0.smithi033.stdout:2677: read oid 4 snap -1 2024-09-21T09:59:51.961 INFO:tasks.rados.rados.0.smithi033.stdout:2677: expect (ObjNum 746 snap 216 seq_num 746) 2024-09-21T09:59:51.961 INFO:tasks.rados.rados.0.smithi033.stdout:2669: expect (ObjNum 746 snap 216 seq_num 746) 2024-09-21T09:59:52.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: pgmap v471: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 299 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 972 KiB/s wr, 10 op/s; 61/624 objects degraded (9.776%) 2024-09-21T09:59:52.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T09:59:52.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T09:59:52.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: pgmap v471: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 299 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 972 KiB/s wr, 10 op/s; 61/624 objects degraded (9.776%) 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.249 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T09:59:52.250 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T09:59:52.251 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T09:59:52.251 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T09:59:52.251 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T09:59:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: pgmap v471: 121 pgs: 37 active+undersized, 13 active+undersized+degraded, 71 active+clean; 299 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 972 KiB/s wr, 10 op/s; 61/624 objects degraded (9.776%) 2024-09-21T09:59:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.0"} : dispatch 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-09-21T09:59:52.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.1"} : dispatch 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.2"} : dispatch 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.3"} : dispatch 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.4"} : dispatch 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.5"} : dispatch 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.6"} : dispatch 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd.7"} : dispatch 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-09-21T09:59:52.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd require-osd-release", "release": "squid"} : dispatch 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:2676: finishing write tid 1 to smithi03385152-47 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:2676: finishing write tid 2 to smithi03385152-47 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:2676: finishing write tid 3 to smithi03385152-47 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:2676: finishing write tid 4 to smithi03385152-47 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:2676: finishing write tid 5 to smithi03385152-47 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:2676: finishing write tid 6 to smithi03385152-47 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1488 (ObjNum 843 snap 248 seq_num 843) dirty exists 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:2676: left oid 47 (ObjNum 843 snap 248 seq_num 843) 2024-09-21T09:59:52.300 INFO:tasks.rados.rados.0.smithi033.stdout:2677: expect (ObjNum 746 snap 216 seq_num 746) 2024-09-21T09:59:52.635 INFO:tasks.rados.rados.0.smithi033.stdout:2669: done (2 left) 2024-09-21T09:59:52.635 INFO:tasks.rados.rados.0.smithi033.stdout:2676: done (1 left) 2024-09-21T09:59:52.636 INFO:tasks.rados.rados.0.smithi033.stdout:2677: done (0 left) 2024-09-21T09:59:52.636 INFO:tasks.rados.rados.0.smithi033.stdout:2678: snap_create 2024-09-21T09:59:52.946 INFO:tasks.rados.rados.0.smithi033.stdout:2678: done (0 left) 2024-09-21T09:59:52.947 INFO:tasks.rados.rados.0.smithi033.stdout:2679: copy_from oid 10 from oid 15 current snap is 249 2024-09-21T09:59:52.947 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 249 2024-09-21T09:59:52.947 INFO:tasks.rados.rados.0.smithi033.stdout:2680: seq_num 844 ranges {2228224=540672,2768896=696320,3465216=475136,3940352=434176} 2024-09-21T09:59:52.962 INFO:tasks.rados.rados.0.smithi033.stdout:2680: writing smithi03385152-37 from 2228224 to 2768896 tid 1 2024-09-21T09:59:52.966 INFO:tasks.rados.rados.0.smithi033.stdout:2680: writing smithi03385152-37 from 2768896 to 3465216 tid 2 2024-09-21T09:59:52.969 INFO:tasks.rados.rados.0.smithi033.stdout:2680: writing smithi03385152-37 from 3465216 to 3940352 tid 3 2024-09-21T09:59:52.971 INFO:tasks.rados.rados.0.smithi033.stdout:2680: writing smithi03385152-37 from 3940352 to 4374528 tid 4 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:2681: copy_from oid 6 from oid 9 current snap is 249 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:2679: finishing copy_from racing read to smithi03385152-10 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:2679: finishing copy_from to smithi03385152-10 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 934 (ObjNum 684 snap 188 seq_num 684) dirty exists 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:2680: finishing write tid 1 to smithi03385152-37 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:2680: finishing write tid 2 to smithi03385152-37 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:2680: finishing write tid 3 to smithi03385152-37 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:2679: done (2 left) 2024-09-21T09:59:52.972 INFO:tasks.rados.rados.0.smithi033.stdout:2682: snap_remove snap 223 2024-09-21T09:59:53.194 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all osd 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[98546]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[98546]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[98546]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[98546]: Upgrade: It appears safe to stop mds.a 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all osd 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[96568]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[96568]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T09:59:53.195 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T09:59:53.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[96568]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:59:53.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-21T09:59:53.196 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:52 smithi033 ceph-mon[96568]: Upgrade: It appears safe to stop mds.a 2024-09-21T09:59:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:52 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all osd 2024-09-21T09:59:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:52 smithi105 ceph-mon[86649]: Upgrade: Setting require_osd_release to 19 squid 2024-09-21T09:59:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:52 smithi105 ceph-mon[86649]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-09-21T09:59:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-09-21T09:59:53.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:52 smithi105 ceph-mon[86649]: osdmap e562: 8 total, 8 up, 8 in 2024-09-21T09:59:53.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["a"]} : dispatch 2024-09-21T09:59:53.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:52 smithi105 ceph-mon[86649]: Upgrade: It appears safe to stop mds.a 2024-09-21T09:59:53.945 INFO:tasks.rados.rados.0.smithi033.stdout:2682: done (2 left) 2024-09-21T09:59:53.945 INFO:tasks.rados.rados.0.smithi033.stdout:2683: delete oid 33 current snap is 249 2024-09-21T09:59:53.945 INFO:tasks.rados.rados.0.smithi033.stdout:2680: finishing write tid 4 to smithi03385152-37 2024-09-21T09:59:53.945 INFO:tasks.rados.rados.0.smithi033.stdout:2680: finishing write tid 5 to smithi03385152-37 2024-09-21T09:59:53.945 INFO:tasks.rados.rados.0.smithi033.stdout:2680: finishing write tid 6 to smithi03385152-37 2024-09-21T09:59:53.945 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1062 (ObjNum 844 snap 249 seq_num 844) dirty exists 2024-09-21T09:59:53.945 INFO:tasks.rados.rados.0.smithi033.stdout:2680: left oid 37 (ObjNum 844 snap 249 seq_num 844) 2024-09-21T09:59:53.945 INFO:tasks.rados.rados.0.smithi033.stdout:2681: finishing copy_from to smithi03385152-6 2024-09-21T09:59:53.946 INFO:tasks.rados.rados.0.smithi033.stdout:2681: got expected ENOENT (src dne) 2024-09-21T09:59:53.946 INFO:tasks.rados.rados.0.smithi033.stdout:2681: finishing copy_from racing read to smithi03385152-6 2024-09-21T09:59:53.955 INFO:tasks.rados.rados.0.smithi033.stdout:2680: done (2 left) 2024-09-21T09:59:53.955 INFO:tasks.rados.rados.0.smithi033.stdout:2681: done (1 left) 2024-09-21T09:59:53.955 INFO:tasks.rados.rados.0.smithi033.stdout:2683: done (0 left) 2024-09-21T09:59:53.956 INFO:tasks.rados.rados.0.smithi033.stdout:2684: rmattr oid 13 current snap is 249 2024-09-21T09:59:53.956 INFO:tasks.rados.rados.0.smithi033.stdout:2684: done (0 left) 2024-09-21T09:59:53.956 INFO:tasks.rados.rados.0.smithi033.stdout:2685: setattr oid 39 current snap is 249 2024-09-21T09:59:53.957 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 249 2024-09-21T09:59:53.957 INFO:tasks.rados.rados.0.smithi033.stdout:2686: seq_num 846 ranges {0=573440,573440=466944,1040384=491520,1531904=548864,2080768=98304} 2024-09-21T09:59:53.962 INFO:tasks.rados.rados.0.smithi033.stdout:2686: writing smithi03385152-43 from 0 to 573440 tid 1 2024-09-21T09:59:53.965 INFO:tasks.rados.rados.0.smithi033.stdout:2686: writing smithi03385152-43 from 573440 to 1040384 tid 2 2024-09-21T09:59:53.969 INFO:tasks.rados.rados.0.smithi033.stdout:2686: writing smithi03385152-43 from 1040384 to 1531904 tid 3 2024-09-21T09:59:53.974 INFO:tasks.rados.rados.0.smithi033.stdout:2686: writing smithi03385152-43 from 1531904 to 2080768 tid 4 2024-09-21T09:59:53.975 INFO:tasks.rados.rados.0.smithi033.stdout:2686: writing smithi03385152-43 from 2080768 to 2179072 tid 5 2024-09-21T09:59:53.975 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 249 2024-09-21T09:59:53.975 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1047 (ObjNum 798 snap 233 seq_num 798) dirty exists 2024-09-21T09:59:53.975 INFO:tasks.rados.rados.0.smithi033.stdout:2686: finishing write tid 1 to smithi03385152-43 2024-09-21T09:59:53.975 INFO:tasks.rados.rados.0.smithi033.stdout:2686: finishing write tid 2 to smithi03385152-43 2024-09-21T09:59:53.975 INFO:tasks.rados.rados.0.smithi033.stdout:2686: finishing write tid 3 to smithi03385152-43 2024-09-21T09:59:53.976 INFO:tasks.rados.rados.0.smithi033.stdout:2687: seq_num 847 ranges {4128768=589824,4718592=753664,5472256=655360,6127616=196608} 2024-09-21T09:59:54.003 INFO:tasks.rados.rados.0.smithi033.stdout:2687: writing smithi03385152-47 from 4128768 to 4718592 tid 1 2024-09-21T09:59:54.008 INFO:tasks.rados.rados.0.smithi033.stdout:2687: writing smithi03385152-47 from 4718592 to 5472256 tid 2 2024-09-21T09:59:54.012 INFO:tasks.rados.rados.0.smithi033.stdout:2687: writing smithi03385152-47 from 5472256 to 6127616 tid 3 2024-09-21T09:59:54.014 INFO:tasks.rados.rados.0.smithi033.stdout:2687: writing smithi03385152-47 from 6127616 to 6324224 tid 4 2024-09-21T09:59:54.014 INFO:tasks.rados.rados.0.smithi033.stdout:2685: done (2 left) 2024-09-21T09:59:54.014 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 249 2024-09-21T09:59:54.014 INFO:tasks.rados.rados.0.smithi033.stdout:2686: finishing write tid 4 to smithi03385152-43 2024-09-21T09:59:54.015 INFO:tasks.rados.rados.0.smithi033.stdout:2686: finishing write tid 5 to smithi03385152-43 2024-09-21T09:59:54.015 INFO:tasks.rados.rados.0.smithi033.stdout:2686: finishing write tid 6 to smithi03385152-43 2024-09-21T09:59:54.015 INFO:tasks.rados.rados.0.smithi033.stdout:2686: finishing write tid 7 to smithi03385152-43 2024-09-21T09:59:54.015 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1068 (ObjNum 846 snap 249 seq_num 846) dirty exists 2024-09-21T09:59:54.015 INFO:tasks.rados.rados.0.smithi033.stdout:2686: left oid 43 (ObjNum 846 snap 249 seq_num 846) 2024-09-21T09:59:54.015 INFO:tasks.rados.rados.0.smithi033.stdout:2687: finishing write tid 1 to smithi03385152-47 2024-09-21T09:59:54.015 INFO:tasks.rados.rados.0.smithi033.stdout:2687: finishing write tid 2 to smithi03385152-47 2024-09-21T09:59:54.015 INFO:tasks.rados.rados.0.smithi033.stdout:2688: seq_num 848 ranges {4349952=606208,4956160=630784,5586944=458752,6045696=516096} 2024-09-21T09:59:54.041 INFO:tasks.rados.rados.0.smithi033.stdout:2688: writing smithi03385152-25 from 4349952 to 4956160 tid 1 2024-09-21T09:59:54.045 INFO:tasks.rados.rados.0.smithi033.stdout:2688: writing smithi03385152-25 from 4956160 to 5586944 tid 2 2024-09-21T09:59:54.047 INFO:tasks.rados.rados.0.smithi033.stdout:2688: writing smithi03385152-25 from 5586944 to 6045696 tid 3 2024-09-21T09:59:54.051 INFO:tasks.rados.rados.0.smithi033.stdout:2688: writing smithi03385152-25 from 6045696 to 6561792 tid 4 2024-09-21T09:59:54.051 INFO:tasks.rados.rados.0.smithi033.stdout:2686: done (2 left) 2024-09-21T09:59:54.051 INFO:tasks.rados.rados.0.smithi033.stdout:2689: rollback oid 12 current snap is 249 2024-09-21T09:59:54.051 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 12 to 245 2024-09-21T09:59:54.051 INFO:tasks.rados.rados.0.smithi033.stdout:2687: finishing write tid 3 to smithi03385152-47 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2687: finishing write tid 4 to smithi03385152-47 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2687: finishing write tid 5 to smithi03385152-47 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2687: finishing write tid 6 to smithi03385152-47 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1493 (ObjNum 847 snap 249 seq_num 847) dirty exists 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2687: left oid 47 (ObjNum 847 snap 249 seq_num 847) 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2688: finishing write tid 1 to smithi03385152-25 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2687: done (2 left) 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2690: delete oid 14 current snap is 249 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2688: finishing write tid 2 to smithi03385152-25 2024-09-21T09:59:54.052 INFO:tasks.rados.rados.0.smithi033.stdout:2688: finishing write tid 3 to smithi03385152-25 2024-09-21T09:59:54.053 INFO:tasks.rados.rados.0.smithi033.stdout:2688: finishing write tid 4 to smithi03385152-25 2024-09-21T09:59:54.054 INFO:tasks.rados.rados.0.smithi033.stdout:2688: finishing write tid 5 to smithi03385152-25 2024-09-21T09:59:54.054 INFO:tasks.rados.rados.0.smithi033.stdout:2689: finishing rollback tid 1 to smithi03385152-12 2024-09-21T09:59:54.054 INFO:tasks.rados.rados.0.smithi033.stdout:2689: finishing rollback tid 2 to smithi03385152-12 2024-09-21T09:59:54.054 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1494 (ObjNum 797 snap 233 seq_num 797) dirty exists 2024-09-21T09:59:54.054 INFO:tasks.rados.rados.0.smithi033.stdout:2689: done (2 left) 2024-09-21T09:59:54.054 INFO:tasks.rados.rados.0.smithi033.stdout:2690: done (1 left) 2024-09-21T09:59:54.054 INFO:tasks.rados.rados.0.smithi033.stdout:2691: snap_create 2024-09-21T09:59:54.055 INFO:tasks.rados.rados.0.smithi033.stdout:2688: finishing write tid 6 to smithi03385152-25 2024-09-21T09:59:54.055 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 951 (ObjNum 848 snap 249 seq_num 848) dirty exists 2024-09-21T09:59:54.055 INFO:tasks.rados.rados.0.smithi033.stdout:2688: left oid 25 (ObjNum 848 snap 249 seq_num 848) 2024-09-21T09:59:54.082 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T09:59:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[98546]: pgmap v474: 121 pgs: 13 peering, 24 active+undersized, 9 active+undersized+degraded, 75 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 3.0 MiB/s wr, 21 op/s; 45/624 objects degraded (7.212%) 2024-09-21T09:59:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[98546]: Health check update: Degraded data redundancy: 45/624 objects degraded (7.212%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[98546]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:59:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:59:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:54.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[96568]: pgmap v474: 121 pgs: 13 peering, 24 active+undersized, 9 active+undersized+degraded, 75 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 3.0 MiB/s wr, 21 op/s; 45/624 objects degraded (7.212%) 2024-09-21T09:59:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[96568]: Health check update: Degraded data redundancy: 45/624 objects degraded (7.212%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[96568]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:59:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:59:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:53 smithi105 ceph-mon[86649]: pgmap v474: 121 pgs: 13 peering, 24 active+undersized, 9 active+undersized+degraded, 75 active+clean; 305 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 3.0 MiB/s wr, 21 op/s; 45/624 objects degraded (7.212%) 2024-09-21T09:59:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:53 smithi105 ceph-mon[86649]: Health check update: Degraded data redundancy: 45/624 objects degraded (7.212%), 9 pgs degraded (PG_DEGRADED) 2024-09-21T09:59:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:53 smithi105 ceph-mon[86649]: osdmap e563: 8 total, 8 up, 8 in 2024-09-21T09:59:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T09:59:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.a", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T09:59:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T09:59:54.670 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (6m) 2m ago 11m 19.2M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (6m) 7s ago 10m 87.7M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (12m) 2m ago 12m 16.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be 3d23d85aab7e 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (12m) 7s ago 12m 22.1M - 17.2.7-1649-gfbd1a6ab fc3ec4a4f6be b0497afc1e27 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (5m) 7s ago 16m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (6m) 2m ago 17m 524M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (5m) 2m ago 18m 56.0M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (4m) 7s ago 16m 52.3M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (5m) 2m ago 16m 47.9M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (6m) 2m ago 10m 11.8M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (6m) 7s ago 10m 11.2M - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (4m) 2m ago 15m 168M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (3m) 2m ago 15m 126M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T09:59:54.671 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (3m) 2m ago 15m 66.9M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 76dbf29565ba 2024-09-21T09:59:54.672 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (2m) 2m ago 14m 11.2M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b85b61817088 2024-09-21T09:59:54.672 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (115s) 7s ago 14m 138M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 01f4f66dbe72 2024-09-21T09:59:54.672 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (80s) 7s ago 13m 161M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4ff1257a8be1 2024-09-21T09:59:54.672 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (46s) 7s ago 13m 84.6M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 132bc834f24d 2024-09-21T09:59:54.672 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (12s) 7s ago 13m 11.9M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 ea10ccd8e9ee 2024-09-21T09:59:54.672 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (5m) 7s ago 11m 52.4M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T09:59:54.948 INFO:tasks.rados.rados.0.smithi033.stdout:2688: done (1 left) 2024-09-21T09:59:54.949 INFO:tasks.rados.rados.0.smithi033.stdout:2691: done (0 left) 2024-09-21T09:59:54.949 INFO:tasks.rados.rados.0.smithi033.stdout:2692: copy_from oid 8 from oid 3 current snap is 250 2024-09-21T09:59:54.949 INFO:tasks.rados.rados.0.smithi033.stdout:2693: setattr oid 13 current snap is 250 2024-09-21T09:59:54.949 INFO:tasks.rados.rados.0.smithi033.stdout:2694: delete oid 23 current snap is 250 2024-09-21T09:59:54.952 INFO:tasks.rados.rados.0.smithi033.stdout:2692: finishing copy_from to smithi03385152-8 2024-09-21T09:59:54.952 INFO:tasks.rados.rados.0.smithi033.stdout:2692: got expected ENOENT (src dne) 2024-09-21T09:59:54.952 INFO:tasks.rados.rados.0.smithi033.stdout:2692: finishing copy_from racing read to smithi03385152-8 2024-09-21T09:59:54.953 INFO:tasks.rados.rados.0.smithi033.stdout:2692: done (2 left) 2024-09-21T09:59:54.953 INFO:tasks.rados.rados.0.smithi033.stdout:2694: done (1 left) 2024-09-21T09:59:54.953 INFO:tasks.rados.rados.0.smithi033.stdout:2695: copy_from oid 30 from oid 35 current snap is 250 2024-09-21T09:59:54.953 INFO:tasks.rados.rados.0.smithi033.stdout:2696: snap_create 2024-09-21T09:59:54.956 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 1921 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-21T09:59:54.962 INFO:tasks.rados.rados.0.smithi033.stdout:2695: finishing copy_from racing read to smithi03385152-30 2024-09-21T09:59:54.963 INFO:tasks.rados.rados.0.smithi033.stdout:2695: finishing copy_from to smithi03385152-30 2024-09-21T09:59:54.963 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 1770 (ObjNum 811 snap 237 seq_num 811) dirty exists 2024-09-21T09:59:55.038 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:59:55.038 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T09:59:55.038 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T09:59:55.038 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:59:55.038 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T09:59:55.038 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 17.2.7-1649-gfbd1a6ab (fbd1a6ab0448f011c60f6376195d5c078b0cf20b) quincy (stable)": 2, 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 13 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T09:59:55.039 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:59:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:54 smithi033 ceph-mon[96568]: Upgrade: Updating mds.a 2024-09-21T09:59:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:54 smithi033 ceph-mon[96568]: Deploying daemon mds.a on smithi033 2024-09-21T09:59:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:54 smithi033 ceph-mon[96568]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:59:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:54 smithi033 ceph-mon[98546]: Upgrade: Updating mds.a 2024-09-21T09:59:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:54 smithi033 ceph-mon[98546]: Deploying daemon mds.a on smithi033 2024-09-21T09:59:55.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:54 smithi033 ceph-mon[98546]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:59:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:54 smithi105 ceph-mon[86649]: Upgrade: Updating mds.a 2024-09-21T09:59:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:54 smithi105 ceph-mon[86649]: Deploying daemon mds.a on smithi033 2024-09-21T09:59:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:54 smithi105 ceph-mon[86649]: osdmap e564: 8 total, 8 up, 8 in 2024-09-21T09:59:55.351 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "osd", 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "13/20 daemons upgraded", 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading mds daemons", 2024-09-21T09:59:55.352 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T09:59:55.353 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T09:59:55.592 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:55 smithi033 systemd[1]: Stopping Ceph mds.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:59:55.950 INFO:tasks.rados.rados.0.smithi033.stdout:2693: done (2 left) 2024-09-21T09:59:55.950 INFO:tasks.rados.rados.0.smithi033.stdout:2695: done (1 left) 2024-09-21T09:59:55.950 INFO:tasks.rados.rados.0.smithi033.stdout:2696: done (0 left) 2024-09-21T09:59:55.951 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 251 2024-09-21T09:59:55.951 INFO:tasks.rados.rados.0.smithi033.stdout:2697: seq_num 851 ranges {5644288=655360,6299648=688128,6987776=679936,7667712=237568} 2024-09-21T09:59:55.953 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[67476]: 2024-09-21T09:59:55.589+0000 7f25a639c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T09:59:55.953 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[67476]: 2024-09-21T09:59:55.589+0000 7f25a639c640 -1 mds.a *** got signal Terminated *** 2024-09-21T09:59:55.985 INFO:tasks.rados.rados.0.smithi033.stdout:2697: writing smithi03385152-12 from 5644288 to 6299648 tid 1 2024-09-21T09:59:55.988 INFO:tasks.rados.rados.0.smithi033.stdout:2697: writing smithi03385152-12 from 6299648 to 6987776 tid 2 2024-09-21T09:59:55.992 INFO:tasks.rados.rados.0.smithi033.stdout:2697: writing smithi03385152-12 from 6987776 to 7667712 tid 3 2024-09-21T09:59:55.994 INFO:tasks.rados.rados.0.smithi033.stdout:2697: writing smithi03385152-12 from 7667712 to 7905280 tid 4 2024-09-21T09:59:55.994 INFO:tasks.rados.rados.0.smithi033.stdout:2698: read oid 33 snap -1 2024-09-21T09:59:55.994 INFO:tasks.rados.rados.0.smithi033.stdout:2698: expect deleted 2024-09-21T09:59:55.994 INFO:tasks.rados.rados.0.smithi033.stdout:2699: rmattr oid 38 current snap is 251 2024-09-21T09:59:55.994 INFO:tasks.rados.rados.0.smithi033.stdout:2697: finishing write tid 1 to smithi03385152-12 2024-09-21T09:59:55.994 INFO:tasks.rados.rados.0.smithi033.stdout:2700: rollback oid 10 current snap is 251 2024-09-21T09:59:55.994 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 245 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:2701: rollback oid 35 current snap is 251 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 35 to 246 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:2702: rollback oid 2 current snap is 251 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 236 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:2703: read oid 17 snap -1 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:2703: expect deleted 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:2704: rollback oid 31 current snap is 251 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 31 to 244 2024-09-21T09:59:55.995 INFO:tasks.rados.rados.0.smithi033.stdout:2705: snap_remove snap 222 2024-09-21T09:59:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[98546]: from='client.44411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[98546]: from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[98546]: pgmap v477: 121 pgs: 13 peering, 3 active+undersized, 105 active+clean; 315 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 MiB/s rd, 5.2 MiB/s wr, 16 op/s; 3.1 MiB/s, 1 objects/s recovering 2024-09-21T09:59:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[98546]: from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[98546]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/624 objects degraded (7.212%), 9 pgs degraded) 2024-09-21T09:59:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T09:59:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[98546]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:59:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3439712242' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[96568]: from='client.44411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[96568]: from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[96568]: pgmap v477: 121 pgs: 13 peering, 3 active+undersized, 105 active+clean; 315 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 MiB/s rd, 5.2 MiB/s wr, 16 op/s; 3.1 MiB/s, 1 objects/s recovering 2024-09-21T09:59:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[96568]: from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[96568]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/624 objects degraded (7.212%), 9 pgs degraded) 2024-09-21T09:59:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T09:59:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[96568]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:59:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:55 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3439712242' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:55 smithi105 ceph-mon[86649]: from='client.44411 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:55 smithi105 ceph-mon[86649]: from='client.44417 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:55 smithi105 ceph-mon[86649]: pgmap v477: 121 pgs: 13 peering, 3 active+undersized, 105 active+clean; 315 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 34 MiB/s rd, 5.2 MiB/s wr, 16 op/s; 3.1 MiB/s, 1 objects/s recovering 2024-09-21T09:59:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:55 smithi105 ceph-mon[86649]: from='client.44423 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:55 smithi105 ceph-mon[86649]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 45/624 objects degraded (7.212%), 9 pgs degraded) 2024-09-21T09:59:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:55 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T09:59:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:55 smithi105 ceph-mon[86649]: osdmap e565: 8 total, 8 up, 8 in 2024-09-21T09:59:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:55 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3439712242' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T09:59:56.969 INFO:tasks.rados.rados.0.smithi033.stdout:2697: finishing write tid 2 to smithi03385152-12 2024-09-21T09:59:56.969 INFO:tasks.rados.rados.0.smithi033.stdout:2697: finishing write tid 3 to smithi03385152-12 2024-09-21T09:59:56.969 INFO:tasks.rados.rados.0.smithi033.stdout:2702: finishing rollback tid 0 to smithi03385152-2 2024-09-21T09:59:56.969 INFO:tasks.rados.rados.0.smithi033.stdout:2702: finishing rollback tid 1 to smithi03385152-2 2024-09-21T09:59:56.969 INFO:tasks.rados.rados.0.smithi033.stdout:2702: finishing rollback tid 2 to smithi03385152-2 2024-09-21T09:59:56.969 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 954 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T09:59:56.969 INFO:tasks.rados.rados.0.smithi033.stdout:2697: finishing write tid 4 to smithi03385152-12 2024-09-21T09:59:56.969 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 1481 (ObjNum 177 snap 0 seq_num 1) dirty exists 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2697: finishing write tid 5 to smithi03385152-12 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2700: finishing rollback tid 0 to smithi03385152-10 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2700: finishing rollback tid 1 to smithi03385152-10 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2700: finishing rollback tid 2 to smithi03385152-10 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1060 (ObjNum 818 snap 238 seq_num 818) dirty exists 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2697: finishing write tid 6 to smithi03385152-12 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1499 (ObjNum 851 snap 251 seq_num 851) dirty exists 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2697: left oid 12 (ObjNum 851 snap 251 seq_num 851) 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2701: finishing rollback tid 0 to smithi03385152-35 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2701: finishing rollback tid 1 to smithi03385152-35 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2701: finishing rollback tid 2 to smithi03385152-35 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 1926 (ObjNum 811 snap 237 seq_num 811) dirty exists 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2704: finishing rollback tid 0 to smithi03385152-31 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2704: finishing rollback tid 1 to smithi03385152-31 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:2704: finishing rollback tid 2 to smithi03385152-31 2024-09-21T09:59:56.970 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1930 (ObjNum 823 snap 240 seq_num 823) dirty exists 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2697: done (8 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2698: done (7 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2699: done (6 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2700: done (5 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2701: done (4 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2702: done (3 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2703: done (2 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2704: done (1 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2705: done (0 left) 2024-09-21T09:59:56.971 INFO:tasks.rados.rados.0.smithi033.stdout:2706: delete oid 27 current snap is 251 2024-09-21T09:59:56.982 INFO:tasks.rados.rados.0.smithi033.stdout:2706: done (0 left) 2024-09-21T09:59:56.983 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 251 2024-09-21T09:59:56.983 INFO:tasks.rados.rados.0.smithi033.stdout:2707: seq_num 853 ranges {5644288=696320,6340608=507904,6848512=647168,7495680=450560} 2024-09-21T09:59:57.018 INFO:tasks.rados.rados.0.smithi033.stdout:2707: writing smithi03385152-34 from 5644288 to 6340608 tid 1 2024-09-21T09:59:57.020 INFO:tasks.rados.rados.0.smithi033.stdout:2707: writing smithi03385152-34 from 6340608 to 6848512 tid 2 2024-09-21T09:59:57.024 INFO:tasks.rados.rados.0.smithi033.stdout:2707: writing smithi03385152-34 from 6848512 to 7495680 tid 3 2024-09-21T09:59:57.027 INFO:tasks.rados.rados.0.smithi033.stdout:2707: writing smithi03385152-34 from 7495680 to 7946240 tid 4 2024-09-21T09:59:57.027 INFO:tasks.rados.rados.0.smithi033.stdout:2708: snap_create 2024-09-21T09:59:57.027 INFO:tasks.rados.rados.0.smithi033.stdout:2707: finishing write tid 1 to smithi03385152-34 2024-09-21T09:59:57.027 INFO:tasks.rados.rados.0.smithi033.stdout:2707: finishing write tid 2 to smithi03385152-34 2024-09-21T09:59:57.027 INFO:tasks.rados.rados.0.smithi033.stdout:2707: finishing write tid 3 to smithi03385152-34 2024-09-21T09:59:57.030 INFO:tasks.rados.rados.0.smithi033.stdout:2707: finishing write tid 4 to smithi03385152-34 2024-09-21T09:59:57.030 INFO:tasks.rados.rados.0.smithi033.stdout:2707: finishing write tid 5 to smithi03385152-34 2024-09-21T09:59:57.031 INFO:tasks.rados.rados.0.smithi033.stdout:2707: finishing write tid 6 to smithi03385152-34 2024-09-21T09:59:57.032 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1562 (ObjNum 853 snap 251 seq_num 853) dirty exists 2024-09-21T09:59:57.032 INFO:tasks.rados.rados.0.smithi033.stdout:2707: left oid 34 (ObjNum 853 snap 251 seq_num 853) 2024-09-21T09:59:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:56 smithi033 ceph-mon[98546]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:56 smithi033 ceph-mon[98546]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:59:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:56 smithi033 ceph-mon[98546]: fsmap cephfs:1 {0=b=up:active} 2024-09-21T09:59:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:56 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:59:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:56 smithi033 ceph-mon[96568]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:56 smithi033 ceph-mon[96568]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:59:57.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:56 smithi033 ceph-mon[96568]: fsmap cephfs:1 {0=b=up:active} 2024-09-21T09:59:57.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:56 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:59:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:56 smithi105 ceph-mon[86649]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T09:59:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:56 smithi105 ceph-mon[86649]: osdmap e566: 8 total, 8 up, 8 in 2024-09-21T09:59:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:56 smithi105 ceph-mon[86649]: fsmap cephfs:1 {0=b=up:active} 2024-09-21T09:59:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:56 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T09:59:57.746 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:57 smithi033 podman[122970]: 2024-09-21 09:59:57.258079863 +0000 UTC m=+1.775493350 container died 3d23d85aab7e3e6b62c2df4b29362b823b2d813335b609ef5d2ebd890de7eeff (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, RELEASE=quincy-fbd1a6a, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-21T09:59:57.966 INFO:tasks.rados.rados.0.smithi033.stdout:2707: done (1 left) 2024-09-21T09:59:57.966 INFO:tasks.rados.rados.0.smithi033.stdout:2708: done (0 left) 2024-09-21T09:59:57.966 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 252 2024-09-21T09:59:57.966 INFO:tasks.rados.rados.0.smithi033.stdout:2709: seq_num 854 ranges {4374528=712704,5087232=794624,5881856=450560,6332416=360448} 2024-09-21T09:59:57.994 INFO:tasks.rados.rados.0.smithi033.stdout:2709: writing smithi03385152-37 from 4374528 to 5087232 tid 1 2024-09-21T09:59:57.998 INFO:tasks.rados.rados.0.smithi033.stdout:2709: writing smithi03385152-37 from 5087232 to 5881856 tid 2 2024-09-21T09:59:58.001 INFO:tasks.rados.rados.0.smithi033.stdout:2709: writing smithi03385152-37 from 5881856 to 6332416 tid 3 2024-09-21T09:59:58.004 INFO:tasks.rados.rados.0.smithi033.stdout:2709: writing smithi03385152-37 from 6332416 to 6692864 tid 4 2024-09-21T09:59:58.004 INFO:tasks.rados.rados.0.smithi033.stdout:2710: rollback oid 2 current snap is 252 2024-09-21T09:59:58.004 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 244 2024-09-21T09:59:58.004 INFO:tasks.rados.rados.0.smithi033.stdout:2709: finishing write tid 1 to smithi03385152-37 2024-09-21T09:59:58.004 INFO:tasks.rados.rados.0.smithi033.stdout:2709: finishing write tid 2 to smithi03385152-37 2024-09-21T09:59:58.004 INFO:tasks.rados.rados.0.smithi033.stdout:2711: delete oid 36 current snap is 252 2024-09-21T09:59:58.007 INFO:tasks.rados.rados.0.smithi033.stdout:2709: finishing write tid 3 to smithi03385152-37 2024-09-21T09:59:58.008 INFO:tasks.rados.rados.0.smithi033.stdout:2709: finishing write tid 4 to smithi03385152-37 2024-09-21T09:59:58.008 INFO:tasks.rados.rados.0.smithi033.stdout:2710: finishing rollback tid 0 to smithi03385152-2 2024-09-21T09:59:58.009 INFO:tasks.rados.rados.0.smithi033.stdout:2710: finishing rollback tid 1 to smithi03385152-2 2024-09-21T09:59:58.010 INFO:tasks.rados.rados.0.smithi033.stdout:2710: finishing rollback tid 2 to smithi03385152-2 2024-09-21T09:59:58.010 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 957 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T09:59:58.010 INFO:tasks.rados.rados.0.smithi033.stdout:2709: finishing write tid 5 to smithi03385152-37 2024-09-21T09:59:58.011 INFO:tasks.rados.rados.0.smithi033.stdout:2710: done (2 left) 2024-09-21T09:59:58.011 INFO:tasks.rados.rados.0.smithi033.stdout:2711: done (1 left) 2024-09-21T09:59:58.011 INFO:tasks.rados.rados.0.smithi033.stdout:2712: snap_remove snap 244 2024-09-21T09:59:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:57 smithi033 ceph-mon[98546]: pgmap v480: 121 pgs: 121 active+clean; 321 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 5.1 MiB/s wr, 16 op/s; 3.1 MiB/s, 1 objects/s recovering 2024-09-21T09:59:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:57 smithi033 ceph-mon[98546]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:59:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:57 smithi033 ceph-mon[96568]: pgmap v480: 121 pgs: 121 active+clean; 321 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 5.1 MiB/s wr, 16 op/s; 3.1 MiB/s, 1 objects/s recovering 2024-09-21T09:59:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:57 smithi033 ceph-mon[96568]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:59:58.247 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:57 smithi033 podman[122970]: 2024-09-21 09:59:57.988425716 +0000 UTC m=+2.505839195 container cleanup 3d23d85aab7e3e6b62c2df4b29362b823b2d813335b609ef5d2ebd890de7eeff (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, GIT_BRANCH=HEAD, RELEASE=quincy-fbd1a6a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2) 2024-09-21T09:59:58.247 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:57 smithi033 bash[122970]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a 2024-09-21T09:59:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:57 smithi105 ceph-mon[86649]: pgmap v480: 121 pgs: 121 active+clean; 321 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 5.1 MiB/s wr, 16 op/s; 3.1 MiB/s, 1 objects/s recovering 2024-09-21T09:59:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:57 smithi105 ceph-mon[86649]: osdmap e567: 8 total, 8 up, 8 in 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:2712: done (1 left) 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:2713: rmattr oid 34 current snap is 252 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:2709: finishing write tid 6 to smithi03385152-37 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1073 (ObjNum 854 snap 252 seq_num 854) dirty exists 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:2709: left oid 37 (ObjNum 854 snap 252 seq_num 854) 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:2709: done (1 left) 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:2714: rollback oid 20 current snap is 252 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 236 2024-09-21T09:59:58.510 INFO:tasks.rados.rados.0.smithi033.stdout:2715: snap_remove snap 250 2024-09-21T09:59:58.746 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:58 smithi033 podman[122989]: 2024-09-21 09:59:58.37362834 +0000 UTC m=+1.108137774 container remove 3d23d85aab7e3e6b62c2df4b29362b823b2d813335b609ef5d2ebd890de7eeff (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, RELEASE=quincy-fbd1a6a) 2024-09-21T09:59:58.746 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:58 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.a.service: Deactivated successfully. 2024-09-21T09:59:58.746 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:58 smithi033 systemd[1]: Stopped Ceph mds.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:59:58.746 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:58 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.a.service: Consumed 1.175s CPU time. 2024-09-21T09:59:59.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:58 smithi033 ceph-mon[98546]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:59:59.127 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:58 smithi033 ceph-mon[98546]: osdmap e569: 8 total, 8 up, 8 in 2024-09-21T09:59:59.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:58 smithi033 ceph-mon[96568]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:59:59.128 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:58 smithi033 ceph-mon[96568]: osdmap e569: 8 total, 8 up, 8 in 2024-09-21T09:59:59.128 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:58 smithi033 systemd[1]: Starting Ceph mds.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T09:59:59.128 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:59 smithi033 podman[123124]: 2024-09-21 09:59:58.909822355 +0000 UTC m=+0.016546799 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T09:59:59.128 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:59 smithi033 podman[123124]: 2024-09-21 09:59:59.01625713 +0000 UTC m=+0.122981563 container create 9bc95e6ef3d05d51eaccf33b583cd4fba918901e61a472cb1f121845798f2645 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T09:59:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:58 smithi105 ceph-mon[86649]: osdmap e568: 8 total, 8 up, 8 in 2024-09-21T09:59:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:58 smithi105 ceph-mon[86649]: osdmap e569: 8 total, 8 up, 8 in 2024-09-21T09:59:59.496 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:59 smithi033 podman[123124]: 2024-09-21 09:59:59.243714403 +0000 UTC m=+0.350438846 container init 9bc95e6ef3d05d51eaccf33b583cd4fba918901e61a472cb1f121845798f2645 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-62da54b, ceph=True, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0) 2024-09-21T09:59:59.496 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:59 smithi033 podman[123124]: 2024-09-21 09:59:59.249055123 +0000 UTC m=+0.355779555 container start 9bc95e6ef3d05d51eaccf33b583cd4fba918901e61a472cb1f121845798f2645 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-21T09:59:59.496 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[123139]: starting mds.a at 2024-09-21T09:59:59.496 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:59 smithi033 bash[123124]: 9bc95e6ef3d05d51eaccf33b583cd4fba918901e61a472cb1f121845798f2645 2024-09-21T09:59:59.496 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 09:59:59 smithi033 systemd[1]: Started Ceph mds.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T09:59:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:2715: done (2 left) 2024-09-21T09:59:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:2716: delete oid 17 current snap is 252 2024-09-21T09:59:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:2714: finishing rollback tid 0 to smithi03385152-20 2024-09-21T09:59:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:2714: finishing rollback tid 1 to smithi03385152-20 2024-09-21T09:59:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:2714: finishing rollback tid 2 to smithi03385152-20 2024-09-21T09:59:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1509 (ObjNum 807 snap 234 seq_num 807) dirty exists 2024-09-21T09:59:59.515 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 1564 (ObjNum 853 snap 251 seq_num 853) dirty exists 2024-09-21T09:59:59.521 INFO:tasks.rados.rados.0.smithi033.stdout:2713: done (2 left) 2024-09-21T09:59:59.521 INFO:tasks.rados.rados.0.smithi033.stdout:2714: done (1 left) 2024-09-21T09:59:59.521 INFO:tasks.rados.rados.0.smithi033.stdout:2716: done (0 left) 2024-09-21T09:59:59.521 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 252 2024-09-21T09:59:59.521 INFO:tasks.rados.rados.0.smithi033.stdout:2717: seq_num 857 ranges {1712128=761856,2473984=442368,2916352=630784,3547136=532480} 2024-09-21T09:59:59.535 INFO:tasks.rados.rados.0.smithi033.stdout:2717: writing smithi03385152-10 from 1712128 to 2473984 tid 1 2024-09-21T09:59:59.537 INFO:tasks.rados.rados.0.smithi033.stdout:2717: writing smithi03385152-10 from 2473984 to 2916352 tid 2 2024-09-21T09:59:59.541 INFO:tasks.rados.rados.0.smithi033.stdout:2717: writing smithi03385152-10 from 2916352 to 3547136 tid 3 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2717: writing smithi03385152-10 from 3547136 to 4079616 tid 4 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2718: read oid 6 snap -1 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2718: expect deleted 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2719: rollback oid 37 current snap is 252 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 237 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2717: finishing write tid 1 to smithi03385152-10 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2717: finishing write tid 2 to smithi03385152-10 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2717: finishing write tid 3 to smithi03385152-10 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2720: read oid 17 snap -1 2024-09-21T09:59:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2720: expect deleted 2024-09-21T09:59:59.546 INFO:tasks.rados.rados.0.smithi033.stdout:2721: snap_remove snap 230 2024-09-21T10:00:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[98546]: pgmap v483: 121 pgs: 121 active+clean; 321 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-21T10:00:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[98546]: osdmap e570: 8 total, 8 up, 8 in 2024-09-21T10:00:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[96568]: pgmap v483: 121 pgs: 121 active+clean; 321 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-21T10:00:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 09:59:59 smithi033 ceph-mon[96568]: osdmap e570: 8 total, 8 up, 8 in 2024-09-21T10:00:00.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:00:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:59 smithi105 ceph-mon[86649]: pgmap v483: 121 pgs: 121 active+clean; 321 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.9 MiB/s wr, 11 op/s 2024-09-21T10:00:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:59 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:59 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:59 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 09:59:59 smithi105 ceph-mon[86649]: osdmap e570: 8 total, 8 up, 8 in 2024-09-21T10:00:00.520 INFO:tasks.rados.rados.0.smithi033.stdout:2721: done (4 left) 2024-09-21T10:00:00.520 INFO:tasks.rados.rados.0.smithi033.stdout:2722: read oid 7 snap -1 2024-09-21T10:00:00.520 INFO:tasks.rados.rados.0.smithi033.stdout:2722: expect (ObjNum 724 snap 205 seq_num 724) 2024-09-21T10:00:00.520 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 252 2024-09-21T10:00:00.520 INFO:tasks.rados.rados.0.smithi033.stdout:2723: seq_num 858 ranges {0=778240,778240=729088,1507328=434176,1941504=442368} 2024-09-21T10:00:00.527 INFO:tasks.rados.rados.0.smithi033.stdout:2723: writing smithi03385152-11 from 0 to 778240 tid 1 2024-09-21T10:00:00.534 INFO:tasks.rados.rados.0.smithi033.stdout:2723: writing smithi03385152-11 from 778240 to 1507328 tid 2 2024-09-21T10:00:00.538 INFO:tasks.rados.rados.0.smithi033.stdout:2723: writing smithi03385152-11 from 1507328 to 1941504 tid 3 2024-09-21T10:00:00.541 INFO:tasks.rados.rados.0.smithi033.stdout:2723: writing smithi03385152-11 from 1941504 to 2383872 tid 4 2024-09-21T10:00:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:2717: finishing write tid 4 to smithi03385152-10 2024-09-21T10:00:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:2719: finishing rollback tid 0 to smithi03385152-37 2024-09-21T10:00:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:2717: finishing write tid 5 to smithi03385152-10 2024-09-21T10:00:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:2719: finishing rollback tid 1 to smithi03385152-37 2024-09-21T10:00:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:2719: finishing rollback tid 2 to smithi03385152-37 2024-09-21T10:00:00.542 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1076 (ObjNum 731 snap 210 seq_num 731) dirty exists 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2717: finishing write tid 6 to smithi03385152-10 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1065 (ObjNum 857 snap 252 seq_num 857) dirty exists 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2717: left oid 10 (ObjNum 857 snap 252 seq_num 857) 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2723: finishing write tid 1 to smithi03385152-11 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2717: done (5 left) 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2718: done (4 left) 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2719: done (3 left) 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2720: done (2 left) 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 252 2024-09-21T10:00:00.543 INFO:tasks.rados.rados.0.smithi033.stdout:2724: seq_num 859 ranges {0=794624,794624=614400,1409024=638976,2048000=352256} 2024-09-21T10:00:00.547 INFO:tasks.rados.rados.0.smithi033.stdout:2724: writing smithi03385152-28 from 0 to 794624 tid 1 2024-09-21T10:00:00.551 INFO:tasks.rados.rados.0.smithi033.stdout:2724: writing smithi03385152-28 from 794624 to 1409024 tid 2 2024-09-21T10:00:00.555 INFO:tasks.rados.rados.0.smithi033.stdout:2724: writing smithi03385152-28 from 1409024 to 2048000 tid 3 2024-09-21T10:00:00.557 INFO:tasks.rados.rados.0.smithi033.stdout:2724: writing smithi03385152-28 from 2048000 to 2400256 tid 4 2024-09-21T10:00:00.557 INFO:tasks.rados.rados.0.smithi033.stdout:2725: setattr oid 27 current snap is 252 2024-09-21T10:00:00.557 INFO:tasks.rados.rados.0.smithi033.stdout:2723: finishing write tid 2 to smithi03385152-11 2024-09-21T10:00:00.557 INFO:tasks.rados.rados.0.smithi033.stdout:2723: finishing write tid 3 to smithi03385152-11 2024-09-21T10:00:00.557 INFO:tasks.rados.rados.0.smithi033.stdout:2723: finishing write tid 4 to smithi03385152-11 2024-09-21T10:00:00.557 INFO:tasks.rados.rados.0.smithi033.stdout:2723: finishing write tid 5 to smithi03385152-11 2024-09-21T10:00:00.558 INFO:tasks.rados.rados.0.smithi033.stdout:2723: finishing write tid 6 to smithi03385152-11 2024-09-21T10:00:00.558 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1081 (ObjNum 858 snap 252 seq_num 858) dirty exists 2024-09-21T10:00:00.558 INFO:tasks.rados.rados.0.smithi033.stdout:2723: left oid 11 (ObjNum 858 snap 252 seq_num 858) 2024-09-21T10:00:00.558 INFO:tasks.rados.rados.0.smithi033.stdout:2722: expect (ObjNum 724 snap 205 seq_num 724) 2024-09-21T10:00:00.807 INFO:tasks.rados.rados.0.smithi033.stdout:2724: finishing write tid 1 to smithi03385152-28 2024-09-21T10:00:00.807 INFO:tasks.rados.rados.0.smithi033.stdout:2724: finishing write tid 2 to smithi03385152-28 2024-09-21T10:00:00.807 INFO:tasks.rados.rados.0.smithi033.stdout:2724: finishing write tid 3 to smithi03385152-28 2024-09-21T10:00:00.807 INFO:tasks.rados.rados.0.smithi033.stdout:2724: finishing write tid 4 to smithi03385152-28 2024-09-21T10:00:00.807 INFO:tasks.rados.rados.0.smithi033.stdout:2724: finishing write tid 5 to smithi03385152-28 2024-09-21T10:00:00.807 INFO:tasks.rados.rados.0.smithi033.stdout:2724: finishing write tid 6 to smithi03385152-28 2024-09-21T10:00:00.807 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 894 (ObjNum 859 snap 252 seq_num 859) dirty exists 2024-09-21T10:00:00.807 INFO:tasks.rados.rados.0.smithi033.stdout:2724: left oid 28 (ObjNum 859 snap 252 seq_num 859) 2024-09-21T10:00:00.808 INFO:tasks.rados.rados.0.smithi033.stdout:2722: done (3 left) 2024-09-21T10:00:00.808 INFO:tasks.rados.rados.0.smithi033.stdout:2723: done (2 left) 2024-09-21T10:00:00.808 INFO:tasks.rados.rados.0.smithi033.stdout:2724: done (1 left) 2024-09-21T10:00:00.808 INFO:tasks.rados.rados.0.smithi033.stdout:2726: copy_from oid 34 from oid 32 current snap is 252 2024-09-21T10:00:00.808 INFO:tasks.rados.rados.0.smithi033.stdout:2727: snap_create 2024-09-21T10:00:00.809 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1610 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-21T10:00:00.832 INFO:tasks.rados.rados.0.smithi033.stdout:2726: finishing copy_from racing read to smithi03385152-34 2024-09-21T10:00:00.839 INFO:tasks.rados.rados.0.smithi033.stdout:2726: finishing copy_from to smithi03385152-34 2024-09-21T10:00:00.839 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 250 (ObjNum 824 snap 241 seq_num 824) dirty exists 2024-09-21T10:00:01.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[98546]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:boot 2024-09-21T10:00:01.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[98546]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T10:00:01.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T10:00:01.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[98546]: overall HEALTH_OK 2024-09-21T10:00:01.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[98546]: pgmap v486: 121 pgs: 121 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T10:00:01.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[98546]: osdmap e571: 8 total, 8 up, 8 in 2024-09-21T10:00:01.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:01.245 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[96568]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:boot 2024-09-21T10:00:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[96568]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T10:00:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T10:00:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[96568]: overall HEALTH_OK 2024-09-21T10:00:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[96568]: pgmap v486: 121 pgs: 121 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T10:00:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[96568]: osdmap e571: 8 total, 8 up, 8 in 2024-09-21T10:00:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-mon[86649]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:boot 2024-09-21T10:00:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-mon[86649]: fsmap cephfs:1 {0=b=up:active} 1 up:standby 2024-09-21T10:00:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T10:00:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-mon[86649]: overall HEALTH_OK 2024-09-21T10:00:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-mon[86649]: pgmap v486: 121 pgs: 121 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-21T10:00:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-mon[86649]: osdmap e571: 8 total, 8 up, 8 in 2024-09-21T10:00:01.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:01.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:01.771 INFO:tasks.rados.rados.0.smithi033.stdout:2725: done (2 left) 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:2726: done (1 left) 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:2727: done (0 left) 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:2728: rollback oid 6 current snap is 253 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 251 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 253 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:2729: seq_num 860 ranges {0=8192} 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:2729: writing smithi03385152-9 from 0 to 8192 tid 1 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:2730: copy_from oid 19 from oid 31 current snap is 253 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:2731: rollback oid 17 current snap is 253 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 249 2024-09-21T10:00:01.772 INFO:tasks.rados.rados.0.smithi033.stdout:2732: delete oid 21 current snap is 253 2024-09-21T10:00:01.774 INFO:tasks.rados.rados.0.smithi033.stdout:2731: finishing rollback tid 1 to smithi03385152-17 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 0 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2728: finishing rollback tid 1 to smithi03385152-6 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1607 (ObjNum 177 snap 0 seq_num 1) dirty dne 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2728: done (4 left) 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2731: done (3 left) 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2732: done (2 left) 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2733: read oid 43 snap -1 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2733: expect (ObjNum 846 snap 249 seq_num 846) 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2734: copy_from oid 3 from oid 18 current snap is 253 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2735: read oid 42 snap -1 2024-09-21T10:00:01.775 INFO:tasks.rados.rados.0.smithi033.stdout:2735: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T10:00:01.776 INFO:tasks.rados.rados.0.smithi033.stdout:2736: read oid 11 snap -1 2024-09-21T10:00:01.776 INFO:tasks.rados.rados.0.smithi033.stdout:2736: expect (ObjNum 858 snap 252 seq_num 858) 2024-09-21T10:00:01.776 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 253 2024-09-21T10:00:01.776 INFO:tasks.rados.rados.0.smithi033.stdout:2737: seq_num 862 ranges {0=40960} 2024-09-21T10:00:01.776 INFO:tasks.rados.rados.0.smithi033.stdout:2737: writing smithi03385152-16 from 0 to 40960 tid 1 2024-09-21T10:00:01.776 INFO:tasks.rados.rados.0.smithi033.stdout:2738: copy_from oid 22 from oid 12 current snap is 253 2024-09-21T10:00:01.776 INFO:tasks.rados.rados.0.smithi033.stdout:2739: snap_remove snap 233 2024-09-21T10:00:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:00:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:00:01] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-09-21T10:00:02.782 INFO:tasks.rados.rados.0.smithi033.stdout:2739: done (8 left) 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 253 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:2740: seq_num 863 ranges {0=57344} 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:2740: writing smithi03385152-46 from 0 to 57344 tid 1 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:2741: delete oid 10 current snap is 253 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:2729: finishing write tid 1 to smithi03385152-9 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:2729: finishing write tid 2 to smithi03385152-9 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:2734: finishing copy_from to smithi03385152-3 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:2734: got expected ENOENT (src dne) 2024-09-21T10:00:02.783 INFO:tasks.rados.rados.0.smithi033.stdout:2734: finishing copy_from racing read to smithi03385152-3 2024-09-21T10:00:02.784 INFO:tasks.rados.rados.0.smithi033.stdout:2735: expect (ObjNum 739 snap 214 seq_num 739) 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2729: finishing write tid 3 to smithi03385152-9 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 1483 (ObjNum 860 snap 253 seq_num 860) dirty exists 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2729: left oid 9 (ObjNum 860 snap 253 seq_num 860) 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2737: finishing write tid 1 to smithi03385152-16 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2729: done (9 left) 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2734: done (8 left) 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2735: done (7 left) 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2741: done (6 left) 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2742: rollback oid 10 current snap is 253 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 246 2024-09-21T10:00:02.800 INFO:tasks.rados.rados.0.smithi033.stdout:2737: finishing write tid 2 to smithi03385152-16 2024-09-21T10:00:02.801 INFO:tasks.rados.rados.0.smithi033.stdout:2737: finishing write tid 3 to smithi03385152-16 2024-09-21T10:00:02.801 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 1964 (ObjNum 862 snap 253 seq_num 862) dirty exists 2024-09-21T10:00:02.801 INFO:tasks.rados.rados.0.smithi033.stdout:2737: left oid 16 (ObjNum 862 snap 253 seq_num 862) 2024-09-21T10:00:02.801 INFO:tasks.rados.rados.0.smithi033.stdout:2733: expect (ObjNum 846 snap 249 seq_num 846) 2024-09-21T10:00:02.861 INFO:tasks.rados.rados.0.smithi033.stdout:2736: expect (ObjNum 858 snap 252 seq_num 858) 2024-09-21T10:00:02.924 INFO:tasks.rados.rados.0.smithi033.stdout:2730: finishing copy_from racing read to smithi03385152-19 2024-09-21T10:00:02.924 INFO:tasks.rados.rados.0.smithi033.stdout:2730: finishing copy_from to smithi03385152-19 2024-09-21T10:00:02.924 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1930 (ObjNum 823 snap 240 seq_num 823) dirty exists 2024-09-21T10:00:02.924 INFO:tasks.rados.rados.0.smithi033.stdout:2738: finishing copy_from racing read to smithi03385152-22 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2738: finishing copy_from to smithi03385152-22 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1499 (ObjNum 851 snap 251 seq_num 851) dirty exists 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2740: finishing write tid 1 to smithi03385152-46 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2740: finishing write tid 2 to smithi03385152-46 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2740: finishing write tid 3 to smithi03385152-46 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1582 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2740: left oid 46 (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2742: finishing rollback tid 1 to smithi03385152-10 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2742: finishing rollback tid 2 to smithi03385152-10 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1933 (ObjNum 818 snap 238 seq_num 818) dirty exists 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2730: done (6 left) 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2733: done (5 left) 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2736: done (4 left) 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2737: done (3 left) 2024-09-21T10:00:02.925 INFO:tasks.rados.rados.0.smithi033.stdout:2738: done (2 left) 2024-09-21T10:00:02.926 INFO:tasks.rados.rados.0.smithi033.stdout:2740: done (1 left) 2024-09-21T10:00:02.926 INFO:tasks.rados.rados.0.smithi033.stdout:2742: done (0 left) 2024-09-21T10:00:02.926 INFO:tasks.rados.rados.0.smithi033.stdout:2743: snap_create 2024-09-21T10:00:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:02 smithi105 ceph-mon[86649]: osdmap e572: 8 total, 8 up, 8 in 2024-09-21T10:00:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:02 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:02 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:02 smithi033 ceph-mon[96568]: osdmap e572: 8 total, 8 up, 8 in 2024-09-21T10:00:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:02 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:03.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:02 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:02 smithi033 ceph-mon[98546]: osdmap e572: 8 total, 8 up, 8 in 2024-09-21T10:00:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:02 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:02 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:03.512 INFO:tasks.rados.rados.0.smithi033.stdout:2743: done (0 left) 2024-09-21T10:00:03.512 INFO:tasks.rados.rados.0.smithi033.stdout:2744: delete oid 44 current snap is 254 2024-09-21T10:00:03.519 INFO:tasks.rados.rados.0.smithi033.stdout:2744: done (0 left) 2024-09-21T10:00:03.519 INFO:tasks.rados.rados.0.smithi033.stdout:2745: read oid 11 snap -1 2024-09-21T10:00:03.519 INFO:tasks.rados.rados.0.smithi033.stdout:2745: expect (ObjNum 858 snap 252 seq_num 858) 2024-09-21T10:00:03.519 INFO:tasks.rados.rados.0.smithi033.stdout:2746: copy_from oid 15 from oid 43 current snap is 254 2024-09-21T10:00:03.519 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 254 2024-09-21T10:00:03.519 INFO:tasks.rados.rados.0.smithi033.stdout:2747: seq_num 866 ranges {6111232=106496} 2024-09-21T10:00:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:2747: writing smithi03385152-32 from 6111232 to 6217728 tid 1 2024-09-21T10:00:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:2748: delete oid 37 current snap is 254 2024-09-21T10:00:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:2745: expect (ObjNum 858 snap 252 seq_num 858) 2024-09-21T10:00:03.620 INFO:tasks.rados.rados.0.smithi033.stdout:2746: finishing copy_from racing read to smithi03385152-15 2024-09-21T10:00:03.620 INFO:tasks.rados.rados.0.smithi033.stdout:2746: finishing copy_from to smithi03385152-15 2024-09-21T10:00:03.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 1068 (ObjNum 846 snap 249 seq_num 846) dirty exists 2024-09-21T10:00:03.620 INFO:tasks.rados.rados.0.smithi033.stdout:2745: done (3 left) 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:2746: done (2 left) 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:2748: done (1 left) 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:2749: setattr oid 29 current snap is 254 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:2747: finishing write tid 1 to smithi03385152-32 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:2747: finishing write tid 2 to smithi03385152-32 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:2747: finishing write tid 3 to smithi03385152-32 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 267 (ObjNum 866 snap 254 seq_num 866) dirty exists 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:2747: left oid 32 (ObjNum 866 snap 254 seq_num 866) 2024-09-21T10:00:03.621 INFO:tasks.rados.rados.0.smithi033.stdout:2747: done (1 left) 2024-09-21T10:00:03.622 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 254 2024-09-21T10:00:03.622 INFO:tasks.rados.rados.0.smithi033.stdout:2750: seq_num 868 ranges {0=139264} 2024-09-21T10:00:03.623 INFO:tasks.rados.rados.0.smithi033.stdout:2750: writing smithi03385152-3 from 0 to 139264 tid 1 2024-09-21T10:00:03.623 INFO:tasks.rados.rados.0.smithi033.stdout:2751: setattr oid 37 current snap is 254 2024-09-21T10:00:03.624 INFO:tasks.rados.rados.0.smithi033.stdout:2752: rollback oid 25 current snap is 254 2024-09-21T10:00:03.624 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 25 to 252 2024-09-21T10:00:03.624 INFO:tasks.rados.rados.0.smithi033.stdout:2753: read oid 34 snap -1 2024-09-21T10:00:03.624 INFO:tasks.rados.rados.0.smithi033.stdout:2753: expect (ObjNum 824 snap 241 seq_num 824) 2024-09-21T10:00:03.624 INFO:tasks.rados.rados.0.smithi033.stdout:2754: read oid 27 snap -1 2024-09-21T10:00:03.624 INFO:tasks.rados.rados.0.smithi033.stdout:2754: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:00:03.624 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 254 2024-09-21T10:00:03.624 INFO:tasks.rados.rados.0.smithi033.stdout:2755: seq_num 869 ranges {0=155648} 2024-09-21T10:00:03.625 INFO:tasks.rados.rados.0.smithi033.stdout:2755: writing smithi03385152-50 from 0 to 155648 tid 1 2024-09-21T10:00:03.625 INFO:tasks.rados.rados.0.smithi033.stdout:2756: snap_remove snap 240 2024-09-21T10:00:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:03 smithi105 ceph-mon[86649]: pgmap v489: 121 pgs: 121 active+clean; 330 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-21T10:00:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:03 smithi105 ceph-mon[86649]: osdmap e573: 8 total, 8 up, 8 in 2024-09-21T10:00:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:03 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:03 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:03 smithi105 ceph-mon[86649]: osdmap e574: 8 total, 8 up, 8 in 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[96568]: pgmap v489: 121 pgs: 121 active+clean; 330 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[96568]: osdmap e573: 8 total, 8 up, 8 in 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[96568]: osdmap e574: 8 total, 8 up, 8 in 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[98546]: pgmap v489: 121 pgs: 121 active+clean; 330 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[98546]: osdmap e573: 8 total, 8 up, 8 in 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:04.031 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:03 smithi033 ceph-mon[98546]: osdmap e574: 8 total, 8 up, 8 in 2024-09-21T10:00:04.516 INFO:tasks.rados.rados.0.smithi033.stdout:2756: done (7 left) 2024-09-21T10:00:04.516 INFO:tasks.rados.rados.0.smithi033.stdout:2757: rmattr oid 42 current snap is 254 2024-09-21T10:00:04.516 INFO:tasks.rados.rados.0.smithi033.stdout:2757: done (7 left) 2024-09-21T10:00:04.516 INFO:tasks.rados.rados.0.smithi033.stdout:2758: snap_remove snap 245 2024-09-21T10:00:04.517 INFO:tasks.rados.rados.0.smithi033.stdout:2754: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:00:04.519 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1083 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2750: finishing write tid 1 to smithi03385152-3 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2752: finishing rollback tid 0 to smithi03385152-25 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2752: finishing rollback tid 1 to smithi03385152-25 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2752: finishing rollback tid 2 to smithi03385152-25 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 960 (ObjNum 848 snap 249 seq_num 848) dirty exists 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2755: finishing write tid 1 to smithi03385152-50 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2755: finishing write tid 2 to smithi03385152-50 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1976 (ObjNum 832 snap 244 seq_num 832) dirty exists 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2755: finishing write tid 3 to smithi03385152-50 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 962 (ObjNum 869 snap 254 seq_num 869) dirty exists 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2755: left oid 50 (ObjNum 869 snap 254 seq_num 869) 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2750: finishing write tid 2 to smithi03385152-3 2024-09-21T10:00:04.520 INFO:tasks.rados.rados.0.smithi033.stdout:2753: expect (ObjNum 824 snap 241 seq_num 824) 2024-09-21T10:00:04.894 INFO:tasks.rados.rados.0.smithi033.stdout:2750: finishing write tid 3 to smithi03385152-3 2024-09-21T10:00:04.894 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1594 (ObjNum 868 snap 254 seq_num 868) dirty exists 2024-09-21T10:00:04.894 INFO:tasks.rados.rados.0.smithi033.stdout:2750: left oid 3 (ObjNum 868 snap 254 seq_num 868) 2024-09-21T10:00:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:05 smithi105 ceph-mon[86649]: pgmap v492: 121 pgs: 121 active+clean; 330 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-21T10:00:05.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:05 smithi105 ceph-mon[86649]: osdmap e575: 8 total, 8 up, 8 in 2024-09-21T10:00:05.785 INFO:tasks.rados.rados.0.smithi033.stdout:2749: done (7 left) 2024-09-21T10:00:05.785 INFO:tasks.rados.rados.0.smithi033.stdout:2750: done (6 left) 2024-09-21T10:00:05.785 INFO:tasks.rados.rados.0.smithi033.stdout:2751: done (5 left) 2024-09-21T10:00:05.785 INFO:tasks.rados.rados.0.smithi033.stdout:2752: done (4 left) 2024-09-21T10:00:05.785 INFO:tasks.rados.rados.0.smithi033.stdout:2753: done (3 left) 2024-09-21T10:00:05.786 INFO:tasks.rados.rados.0.smithi033.stdout:2754: done (2 left) 2024-09-21T10:00:05.786 INFO:tasks.rados.rados.0.smithi033.stdout:2755: done (1 left) 2024-09-21T10:00:05.786 INFO:tasks.rados.rados.0.smithi033.stdout:2758: done (0 left) 2024-09-21T10:00:05.786 INFO:tasks.rados.rados.0.smithi033.stdout:2759: rollback oid 11 current snap is 254 2024-09-21T10:00:05.786 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 11 to 236 2024-09-21T10:00:05.786 INFO:tasks.rados.rados.0.smithi033.stdout:2760: delete oid 36 current snap is 254 2024-09-21T10:00:05.797 INFO:tasks.rados.rados.0.smithi033.stdout:2759: finishing rollback tid 0 to smithi03385152-11 2024-09-21T10:00:05.797 INFO:tasks.rados.rados.0.smithi033.stdout:2760: done (1 left) 2024-09-21T10:00:05.797 INFO:tasks.rados.rados.0.smithi033.stdout:2761: copy_from oid 1 from oid 9 current snap is 254 2024-09-21T10:00:05.797 INFO:tasks.rados.rados.0.smithi033.stdout:2762: snap_remove snap 253 2024-09-21T10:00:05.916 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:05 smithi033 ceph-mon[98546]: pgmap v492: 121 pgs: 121 active+clean; 330 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-21T10:00:05.916 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:05 smithi033 ceph-mon[98546]: osdmap e575: 8 total, 8 up, 8 in 2024-09-21T10:00:05.916 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:05 smithi033 ceph-mon[96568]: pgmap v492: 121 pgs: 121 active+clean; 330 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 2.6 MiB/s wr, 17 op/s 2024-09-21T10:00:05.916 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:05 smithi033 ceph-mon[96568]: osdmap e575: 8 total, 8 up, 8 in 2024-09-21T10:00:06.799 INFO:tasks.rados.rados.0.smithi033.stdout:2762: done (2 left) 2024-09-21T10:00:06.799 INFO:tasks.rados.rados.0.smithi033.stdout:2763: setattr oid 38 current snap is 254 2024-09-21T10:00:06.799 INFO:tasks.rados.rados.0.smithi033.stdout:2759: finishing rollback tid 1 to smithi03385152-11 2024-09-21T10:00:06.799 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1085 (ObjNum 49 snap 0 seq_num 1498634073) dirty dne 2024-09-21T10:00:06.800 INFO:tasks.rados.rados.0.smithi033.stdout:2761: finishing copy_from racing read to smithi03385152-1 2024-09-21T10:00:06.800 INFO:tasks.rados.rados.0.smithi033.stdout:2761: finishing copy_from to smithi03385152-1 2024-09-21T10:00:06.800 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1483 (ObjNum 860 snap 253 seq_num 860) dirty exists 2024-09-21T10:00:06.801 INFO:tasks.rados.rados.0.smithi033.stdout:2759: done (2 left) 2024-09-21T10:00:06.801 INFO:tasks.rados.rados.0.smithi033.stdout:2761: done (1 left) 2024-09-21T10:00:06.801 INFO:tasks.rados.rados.0.smithi033.stdout:2764: snap_create 2024-09-21T10:00:06.807 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 1484 (ObjNum 97 snap 0 seq_num 91100816) dirty exists 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: osdmap e576: 8 total, 8 up, 8 in 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:06 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T10:00:07.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: osdmap e576: 8 total, 8 up, 8 in 2024-09-21T10:00:07.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: osdmap e576: 8 total, 8 up, 8 in 2024-09-21T10:00:07.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:07.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:06 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds ok-to-stop", "ids": ["b"]} : dispatch 2024-09-21T10:00:07.800 INFO:tasks.rados.rados.0.smithi033.stdout:2763: done (1 left) 2024-09-21T10:00:07.800 INFO:tasks.rados.rados.0.smithi033.stdout:2764: done (0 left) 2024-09-21T10:00:07.800 INFO:tasks.rados.rados.0.smithi033.stdout:2765: snap_remove snap 204 2024-09-21T10:00:08.158 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:07 smithi105 ceph-mon[86649]: Detected new or changed devices on smithi033 2024-09-21T10:00:08.158 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:07 smithi105 ceph-mon[86649]: Upgrade: It appears safe to stop mds.b 2024-09-21T10:00:08.158 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:07 smithi105 ceph-mon[86649]: pgmap v495: 121 pgs: 121 active+clean; 343 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.4 MiB/s wr, 31 op/s 2024-09-21T10:00:08.158 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:07 smithi105 ceph-mon[86649]: osdmap e577: 8 total, 8 up, 8 in 2024-09-21T10:00:08.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:07 smithi033 ceph-mon[98546]: Detected new or changed devices on smithi033 2024-09-21T10:00:08.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:07 smithi033 ceph-mon[98546]: Upgrade: It appears safe to stop mds.b 2024-09-21T10:00:08.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:07 smithi033 ceph-mon[98546]: pgmap v495: 121 pgs: 121 active+clean; 343 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.4 MiB/s wr, 31 op/s 2024-09-21T10:00:08.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:07 smithi033 ceph-mon[98546]: osdmap e577: 8 total, 8 up, 8 in 2024-09-21T10:00:08.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:07 smithi033 ceph-mon[96568]: Detected new or changed devices on smithi033 2024-09-21T10:00:08.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:07 smithi033 ceph-mon[96568]: Upgrade: It appears safe to stop mds.b 2024-09-21T10:00:08.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:07 smithi033 ceph-mon[96568]: pgmap v495: 121 pgs: 121 active+clean; 343 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.4 MiB/s wr, 31 op/s 2024-09-21T10:00:08.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:07 smithi033 ceph-mon[96568]: osdmap e577: 8 total, 8 up, 8 in 2024-09-21T10:00:08.517 INFO:tasks.rados.rados.0.smithi033.stdout:2765: done (0 left) 2024-09-21T10:00:08.517 INFO:tasks.rados.rados.0.smithi033.stdout:2766: delete oid 29 current snap is 255 2024-09-21T10:00:08.533 INFO:tasks.rados.rados.0.smithi033.stdout:2766: done (0 left) 2024-09-21T10:00:08.533 INFO:tasks.rados.rados.0.smithi033.stdout:2767: copy_from oid 14 from oid 22 current snap is 255 2024-09-21T10:00:08.534 INFO:tasks.rados.rados.0.smithi033.stdout:2768: delete oid 40 current snap is 255 2024-09-21T10:00:08.538 INFO:tasks.rados.rados.0.smithi033.stdout:2768: done (1 left) 2024-09-21T10:00:08.539 INFO:tasks.rados.rados.0.smithi033.stdout:2769: delete oid 21 current snap is 255 2024-09-21T10:00:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:2769: done (1 left) 2024-09-21T10:00:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:2770: rmattr oid 28 current snap is 255 2024-09-21T10:00:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:2770: done (1 left) 2024-09-21T10:00:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:2771: read oid 7 snap 251 2024-09-21T10:00:08.542 INFO:tasks.rados.rados.0.smithi033.stdout:2771: expect (ObjNum 724 snap 205 seq_num 724) 2024-09-21T10:00:08.543 INFO:tasks.rados.rados.0.smithi033.stdout:2772: delete oid 28 current snap is 255 2024-09-21T10:00:08.544 INFO:tasks.rados.rados.0.smithi033.stdout:2772: done (2 left) 2024-09-21T10:00:08.544 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 255 2024-09-21T10:00:08.545 INFO:tasks.rados.rados.0.smithi033.stdout:2773: seq_num 875 ranges {139264=262144} 2024-09-21T10:00:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:2773: writing smithi03385152-3 from 139264 to 401408 tid 1 2024-09-21T10:00:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:2774: delete oid 27 current snap is 255 2024-09-21T10:00:08.551 INFO:tasks.rados.rados.0.smithi033.stdout:2774: done (3 left) 2024-09-21T10:00:08.551 INFO:tasks.rados.rados.0.smithi033.stdout:2775: snap_remove snap 246 2024-09-21T10:00:09.158 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:08 smithi105 ceph-mon[86649]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T10:00:09.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:08 smithi105 ceph-mon[86649]: Upgrade: Updating mds.b 2024-09-21T10:00:09.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:09.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:00:09.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:08 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:09.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:08 smithi105 ceph-mon[86649]: Deploying daemon mds.b on smithi105 2024-09-21T10:00:09.159 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:08 smithi105 ceph-mon[86649]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T10:00:09.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[98546]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T10:00:09.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[98546]: Upgrade: Updating mds.b 2024-09-21T10:00:09.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[98546]: Deploying daemon mds.b on smithi105 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[98546]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[96568]: osdmap e578: 8 total, 8 up, 8 in 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[96568]: Upgrade: Updating mds.b 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.b", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[96568]: Deploying daemon mds.b on smithi105 2024-09-21T10:00:09.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:08 smithi033 ceph-mon[96568]: osdmap e579: 8 total, 8 up, 8 in 2024-09-21T10:00:09.525 INFO:tasks.rados.rados.0.smithi033.stdout:2771: expect (ObjNum 724 snap 205 seq_num 724) 2024-09-21T10:00:09.715 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:09 smithi105 systemd[1]: Stopping Ceph mds.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:00:09.775 INFO:tasks.rados.rados.0.smithi033.stdout:2767: finishing copy_from racing read to smithi03385152-14 2024-09-21T10:00:09.775 INFO:tasks.rados.rados.0.smithi033.stdout:2767: finishing copy_from to smithi03385152-14 2024-09-21T10:00:09.775 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1499 (ObjNum 851 snap 251 seq_num 851) dirty exists 2024-09-21T10:00:09.775 INFO:tasks.rados.rados.0.smithi033.stdout:2767: done (3 left) 2024-09-21T10:00:09.775 INFO:tasks.rados.rados.0.smithi033.stdout:2771: done (2 left) 2024-09-21T10:00:09.776 INFO:tasks.rados.rados.0.smithi033.stdout:2775: done (1 left) 2024-09-21T10:00:09.776 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 255 2024-09-21T10:00:09.776 INFO:tasks.rados.rados.0.smithi033.stdout:2776: seq_num 877 ranges {385024=294912} 2024-09-21T10:00:09.779 INFO:tasks.rados.rados.0.smithi033.stdout:2776: writing smithi03385152-8 from 385024 to 679936 tid 1 2024-09-21T10:00:09.779 INFO:tasks.rados.rados.0.smithi033.stdout:2777: read oid 49 snap -1 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2777: expect (ObjNum 795 snap 233 seq_num 795) 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2778: rollback oid 24 current snap is 255 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 237 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2773: finishing write tid 1 to smithi03385152-3 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2773: finishing write tid 2 to smithi03385152-3 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2773: finishing write tid 3 to smithi03385152-3 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1618 (ObjNum 875 snap 255 seq_num 875) dirty exists 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2773: left oid 3 (ObjNum 875 snap 255 seq_num 875) 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2773: done (3 left) 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2779: read oid 15 snap -1 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2779: expect (ObjNum 846 snap 249 seq_num 846) 2024-09-21T10:00:09.780 INFO:tasks.rados.rados.0.smithi033.stdout:2780: snap_create 2024-09-21T10:00:09.781 INFO:tasks.rados.rados.0.smithi033.stdout:2778: finishing rollback tid 1 to smithi03385152-24 2024-09-21T10:00:09.781 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 0 (ObjNum 65 snap 0 seq_num 72668112) dirty dne 2024-09-21T10:00:09.786 INFO:tasks.rados.rados.0.smithi033.stdout:2777: expect (ObjNum 795 snap 233 seq_num 795) 2024-09-21T10:00:09.820 INFO:tasks.rados.rados.0.smithi033.stdout:2776: finishing write tid 1 to smithi03385152-8 2024-09-21T10:00:09.821 INFO:tasks.rados.rados.0.smithi033.stdout:2776: finishing write tid 2 to smithi03385152-8 2024-09-21T10:00:09.821 INFO:tasks.rados.rados.0.smithi033.stdout:2779: expect (ObjNum 846 snap 249 seq_num 846) 2024-09-21T10:00:09.880 INFO:tasks.rados.rados.0.smithi033.stdout:2776: finishing write tid 3 to smithi03385152-8 2024-09-21T10:00:09.880 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 1485 (ObjNum 877 snap 255 seq_num 877) dirty exists 2024-09-21T10:00:09.880 INFO:tasks.rados.rados.0.smithi033.stdout:2776: left oid 8 (ObjNum 877 snap 255 seq_num 877) 2024-09-21T10:00:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: pgmap v498: 121 pgs: 121 active+clean; 344 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.5 MiB/s wr, 21 op/s 2024-09-21T10:00:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T10:00:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:00:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:00:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T10:00:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-21T10:00:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[98546]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a[96545]: 2024-09-21T10:00:09.714+0000 7fadcba59640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: pgmap v498: 121 pgs: 121 active+clean; 344 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.5 MiB/s wr, 21 op/s 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T10:00:09.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:09 smithi033 ceph-mon[96568]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-21T10:00:10.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b[59286]: 2024-09-21T10:00:09.712+0000 7fe23afa9640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:00:10.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b[59286]: 2024-09-21T10:00:09.712+0000 7fe23afa9640 -1 mds.b *** got signal Terminated *** 2024-09-21T10:00:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: pgmap v498: 121 pgs: 121 active+clean; 344 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.5 MiB/s wr, 21 op/s 2024-09-21T10:00:10.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: osdmap e580: 8 total, 8 up, 8 in 2024-09-21T10:00:10.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:00:10.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:00:10.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: osdmap e581: 8 total, 8 up, 8 in 2024-09-21T10:00:10.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: Standby daemon mds.a assigned to filesystem cephfs as rank 0 2024-09-21T10:00:10.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:00:10.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T10:00:10.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:09 smithi105 ceph-mon[86649]: fsmap cephfs:1/1 {0=a=up:replay} 2024-09-21T10:00:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:00:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:00:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:10.733 INFO:tasks.rados.rados.0.smithi033.stdout:2776: done (4 left) 2024-09-21T10:00:10.734 INFO:tasks.rados.rados.0.smithi033.stdout:2777: done (3 left) 2024-09-21T10:00:10.734 INFO:tasks.rados.rados.0.smithi033.stdout:2778: done (2 left) 2024-09-21T10:00:10.734 INFO:tasks.rados.rados.0.smithi033.stdout:2779: done (1 left) 2024-09-21T10:00:10.734 INFO:tasks.rados.rados.0.smithi033.stdout:2780: done (0 left) 2024-09-21T10:00:10.734 INFO:tasks.rados.rados.0.smithi033.stdout:2781: read oid 33 snap -1 2024-09-21T10:00:10.734 INFO:tasks.rados.rados.0.smithi033.stdout:2781: expect deleted 2024-09-21T10:00:10.734 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 256 2024-09-21T10:00:10.734 INFO:tasks.rados.rados.0.smithi033.stdout:2782: seq_num 878 ranges {2179072=311296} 2024-09-21T10:00:10.747 INFO:tasks.rados.rados.0.smithi033.stdout:2782: writing smithi03385152-43 from 2179072 to 2490368 tid 1 2024-09-21T10:00:10.747 INFO:tasks.rados.rados.0.smithi033.stdout:2783: read oid 8 snap -1 2024-09-21T10:00:10.747 INFO:tasks.rados.rados.0.smithi033.stdout:2783: expect (ObjNum 877 snap 255 seq_num 877) 2024-09-21T10:00:10.747 INFO:tasks.rados.rados.0.smithi033.stdout:2784: rollback oid 1 current snap is 256 2024-09-21T10:00:10.747 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 207 2024-09-21T10:00:10.747 INFO:tasks.rados.rados.0.smithi033.stdout:2781: done (3 left) 2024-09-21T10:00:10.747 INFO:tasks.rados.rados.0.smithi033.stdout:2785: snap_remove snap 254 2024-09-21T10:00:11.739 INFO:tasks.rados.rados.0.smithi033.stdout:2782: finishing write tid 1 to smithi03385152-43 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:2782: finishing write tid 2 to smithi03385152-43 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:2784: finishing rollback tid 0 to smithi03385152-1 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:2784: finishing rollback tid 1 to smithi03385152-1 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:2784: finishing rollback tid 2 to smithi03385152-1 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1488 (ObjNum 723 snap 205 seq_num 723) dirty exists 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:2782: finishing write tid 3 to smithi03385152-43 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1087 (ObjNum 878 snap 256 seq_num 878) dirty exists 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:2782: left oid 43 (ObjNum 878 snap 256 seq_num 878) 2024-09-21T10:00:11.740 INFO:tasks.rados.rados.0.smithi033.stdout:2783: expect (ObjNum 877 snap 255 seq_num 877) 2024-09-21T10:00:11.767 INFO:tasks.rados.rados.0.smithi033.stdout:2782: done (3 left) 2024-09-21T10:00:11.767 INFO:tasks.rados.rados.0.smithi033.stdout:2783: done (2 left) 2024-09-21T10:00:11.767 INFO:tasks.rados.rados.0.smithi033.stdout:2784: done (1 left) 2024-09-21T10:00:11.768 INFO:tasks.rados.rados.0.smithi033.stdout:2785: done (0 left) 2024-09-21T10:00:11.768 INFO:tasks.rados.rados.0.smithi033.stdout:2786: snap_remove snap 236 2024-09-21T10:00:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:11 smithi105 ceph-mon[86649]: pgmap v502: 121 pgs: 121 active+clean; 345 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 284 B/s rd, 29 KiB/s wr, 8 op/s 2024-09-21T10:00:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:11 smithi105 ceph-mon[86649]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:reconnect 2024-09-21T10:00:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:11 smithi105 ceph-mon[86649]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-21T10:00:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:11 smithi105 ceph-mon[86649]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T10:00:11.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:11 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T10:00:11.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:00:11] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-09-21T10:00:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[98546]: pgmap v502: 121 pgs: 121 active+clean; 345 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 284 B/s rd, 29 KiB/s wr, 8 op/s 2024-09-21T10:00:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[98546]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:reconnect 2024-09-21T10:00:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[98546]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-21T10:00:11.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[98546]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T10:00:11.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T10:00:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[96568]: pgmap v502: 121 pgs: 121 active+clean; 345 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 284 B/s rd, 29 KiB/s wr, 8 op/s 2024-09-21T10:00:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[96568]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:reconnect 2024-09-21T10:00:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[96568]: fsmap cephfs:1/1 {0=a=up:reconnect} 2024-09-21T10:00:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[96568]: osdmap e582: 8 total, 8 up, 8 in 2024-09-21T10:00:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:11 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T10:00:12.746 INFO:tasks.rados.rados.0.smithi033.stdout:2786: done (0 left) 2024-09-21T10:00:12.746 INFO:tasks.rados.rados.0.smithi033.stdout:2787: rmattr oid 5 current snap is 256 2024-09-21T10:00:12.746 INFO:tasks.rados.rados.0.smithi033.stdout:2787: done (0 left) 2024-09-21T10:00:12.746 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 256 2024-09-21T10:00:12.746 INFO:tasks.rados.rados.0.smithi033.stdout:2788: seq_num 879 ranges {0=327680} 2024-09-21T10:00:12.748 INFO:tasks.rados.rados.0.smithi033.stdout:2788: writing smithi03385152-28 from 0 to 327680 tid 1 2024-09-21T10:00:12.748 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 256 2024-09-21T10:00:12.748 INFO:tasks.rados.rados.0.smithi033.stdout:2789: seq_num 880 ranges {0=344064} 2024-09-21T10:00:12.751 INFO:tasks.rados.rados.0.smithi033.stdout:2789: writing smithi03385152-29 from 0 to 344064 tid 1 2024-09-21T10:00:12.751 INFO:tasks.rados.rados.0.smithi033.stdout:2790: read oid 19 snap -1 2024-09-21T10:00:12.751 INFO:tasks.rados.rados.0.smithi033.stdout:2790: expect (ObjNum 823 snap 240 seq_num 823) 2024-09-21T10:00:12.751 INFO:tasks.rados.rados.0.smithi033.stdout:2791: snap_remove snap 255 2024-09-21T10:00:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[96568]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:rejoin 2024-09-21T10:00:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[96568]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-21T10:00:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[96568]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-21T10:00:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[96568]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T10:00:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[96568]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:00:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:00:12.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[98546]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:rejoin 2024-09-21T10:00:12.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[98546]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-21T10:00:12.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[98546]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-21T10:00:12.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[98546]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T10:00:12.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[98546]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:00:12.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:12 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:00:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:12 smithi105 ceph-mon[86649]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:rejoin 2024-09-21T10:00:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:12 smithi105 ceph-mon[86649]: fsmap cephfs:1/1 {0=a=up:rejoin} 2024-09-21T10:00:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:12 smithi105 ceph-mon[86649]: daemon mds.a is now active in filesystem cephfs as rank 0 2024-09-21T10:00:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:12 smithi105 ceph-mon[86649]: osdmap e583: 8 total, 8 up, 8 in 2024-09-21T10:00:13.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:12 smithi105 ceph-mon[86649]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:00:13.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:12 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:00:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2788: finishing write tid 1 to smithi03385152-28 2024-09-21T10:00:13.516 INFO:tasks.rados.rados.0.smithi033.stdout:2788: finishing write tid 2 to smithi03385152-28 2024-09-21T10:00:13.517 INFO:tasks.rados.rados.0.smithi033.stdout:2788: finishing write tid 3 to smithi03385152-28 2024-09-21T10:00:13.517 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 897 (ObjNum 879 snap 256 seq_num 879) dirty exists 2024-09-21T10:00:13.517 INFO:tasks.rados.rados.0.smithi033.stdout:2788: left oid 28 (ObjNum 879 snap 256 seq_num 879) 2024-09-21T10:00:13.518 INFO:tasks.rados.rados.0.smithi033.stdout:2789: finishing write tid 1 to smithi03385152-29 2024-09-21T10:00:13.518 INFO:tasks.rados.rados.0.smithi033.stdout:2789: finishing write tid 2 to smithi03385152-29 2024-09-21T10:00:13.518 INFO:tasks.rados.rados.0.smithi033.stdout:2789: finishing write tid 3 to smithi03385152-29 2024-09-21T10:00:13.518 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2031 (ObjNum 880 snap 256 seq_num 880) dirty exists 2024-09-21T10:00:13.518 INFO:tasks.rados.rados.0.smithi033.stdout:2789: left oid 29 (ObjNum 880 snap 256 seq_num 880) 2024-09-21T10:00:13.518 INFO:tasks.rados.rados.0.smithi033.stdout:2790: expect (ObjNum 823 snap 240 seq_num 823) 2024-09-21T10:00:13.649 INFO:tasks.rados.rados.0.smithi033.stdout:2788: done (3 left) 2024-09-21T10:00:13.649 INFO:tasks.rados.rados.0.smithi033.stdout:2789: done (2 left) 2024-09-21T10:00:13.649 INFO:tasks.rados.rados.0.smithi033.stdout:2790: done (1 left) 2024-09-21T10:00:13.649 INFO:tasks.rados.rados.0.smithi033.stdout:2791: done (0 left) 2024-09-21T10:00:13.649 INFO:tasks.rados.rados.0.smithi033.stdout:2792: setattr oid 19 current snap is 256 2024-09-21T10:00:13.649 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 256 2024-09-21T10:00:13.649 INFO:tasks.rados.rados.0.smithi033.stdout:2793: seq_num 881 ranges {1327104=360448} 2024-09-21T10:00:13.659 INFO:tasks.rados.rados.0.smithi033.stdout:2793: writing smithi03385152-49 from 1327104 to 1687552 tid 1 2024-09-21T10:00:13.659 INFO:tasks.rados.rados.0.smithi033.stdout:2794: snap_remove snap 237 2024-09-21T10:00:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[96568]: pgmap v505: 121 pgs: 121 active+clean; 349 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T10:00:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[96568]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:active 2024-09-21T10:00:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[96568]: fsmap cephfs:1 {0=a=up:active} 2024-09-21T10:00:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[96568]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T10:00:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[96568]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T10:00:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[98546]: pgmap v505: 121 pgs: 121 active+clean; 349 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T10:00:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[98546]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:active 2024-09-21T10:00:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[98546]: fsmap cephfs:1 {0=a=up:active} 2024-09-21T10:00:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[98546]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T10:00:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:13 smithi033 ceph-mon[98546]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T10:00:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:13 smithi105 ceph-mon[86649]: pgmap v505: 121 pgs: 121 active+clean; 349 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 2.1 MiB/s wr, 14 op/s 2024-09-21T10:00:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:13 smithi105 ceph-mon[86649]: mds.? [v2:172.21.15.33:6834/1626081805,v1:172.21.15.33:6835/1626081805] up:active 2024-09-21T10:00:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:13 smithi105 ceph-mon[86649]: fsmap cephfs:1 {0=a=up:active} 2024-09-21T10:00:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:13 smithi105 ceph-mon[86649]: osdmap e584: 8 total, 8 up, 8 in 2024-09-21T10:00:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:13 smithi105 ceph-mon[86649]: osdmap e585: 8 total, 8 up, 8 in 2024-09-21T10:00:14.517 INFO:tasks.rados.rados.0.smithi033.stdout:2794: done (2 left) 2024-09-21T10:00:14.517 INFO:tasks.rados.rados.0.smithi033.stdout:2795: snap_remove snap 251 2024-09-21T10:00:15.246 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:00:14 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[123139]: 2024-09-21T10:00:14.984+0000 7f83a6913640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-21T10:00:15.524 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1934 (ObjNum 823 snap 240 seq_num 823) dirty exists 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:2793: finishing write tid 1 to smithi03385152-49 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:2793: finishing write tid 2 to smithi03385152-49 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:2793: finishing write tid 3 to smithi03385152-49 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1501 (ObjNum 881 snap 256 seq_num 881) dirty exists 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:2793: left oid 49 (ObjNum 881 snap 256 seq_num 881) 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:2792: done (2 left) 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:2793: done (1 left) 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:2795: done (0 left) 2024-09-21T10:00:15.525 INFO:tasks.rados.rados.0.smithi033.stdout:2796: delete oid 28 current snap is 256 2024-09-21T10:00:15.528 INFO:tasks.rados.rados.0.smithi033.stdout:2796: done (0 left) 2024-09-21T10:00:15.529 INFO:tasks.rados.rados.0.smithi033.stdout:2797: delete oid 19 current snap is 256 2024-09-21T10:00:15.537 INFO:tasks.rados.rados.0.smithi033.stdout:2797: done (0 left) 2024-09-21T10:00:15.537 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 256 2024-09-21T10:00:15.537 INFO:tasks.rados.rados.0.smithi033.stdout:2798: seq_num 884 ranges {0=409600} 2024-09-21T10:00:15.539 INFO:tasks.rados.rados.0.smithi033.stdout:2798: writing smithi03385152-18 from 0 to 409600 tid 1 2024-09-21T10:00:15.539 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 256 2024-09-21T10:00:15.540 INFO:tasks.rados.rados.0.smithi033.stdout:2799: seq_num 885 ranges {7905280=425984} 2024-09-21T10:00:15.585 INFO:tasks.rados.rados.0.smithi033.stdout:2799: writing smithi03385152-14 from 7905280 to 8331264 tid 1 2024-09-21T10:00:15.586 INFO:tasks.rados.rados.0.smithi033.stdout:2800: delete oid 35 current snap is 256 2024-09-21T10:00:15.586 INFO:tasks.rados.rados.0.smithi033.stdout:2798: finishing write tid 1 to smithi03385152-18 2024-09-21T10:00:15.586 INFO:tasks.rados.rados.0.smithi033.stdout:2798: finishing write tid 2 to smithi03385152-18 2024-09-21T10:00:15.586 INFO:tasks.rados.rados.0.smithi033.stdout:2798: finishing write tid 3 to smithi03385152-18 2024-09-21T10:00:15.586 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1009 (ObjNum 884 snap 256 seq_num 884) dirty exists 2024-09-21T10:00:15.586 INFO:tasks.rados.rados.0.smithi033.stdout:2798: left oid 18 (ObjNum 884 snap 256 seq_num 884) 2024-09-21T10:00:15.589 INFO:tasks.rados.rados.0.smithi033.stdout:2799: finishing write tid 1 to smithi03385152-14 2024-09-21T10:00:15.590 INFO:tasks.rados.rados.0.smithi033.stdout:2799: finishing write tid 2 to smithi03385152-14 2024-09-21T10:00:15.592 INFO:tasks.rados.rados.0.smithi033.stdout:2799: finishing write tid 3 to smithi03385152-14 2024-09-21T10:00:15.592 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1642 (ObjNum 885 snap 256 seq_num 885) dirty exists 2024-09-21T10:00:15.592 INFO:tasks.rados.rados.0.smithi033.stdout:2799: left oid 14 (ObjNum 885 snap 256 seq_num 885) 2024-09-21T10:00:15.592 INFO:tasks.rados.rados.0.smithi033.stdout:2798: done (2 left) 2024-09-21T10:00:15.592 INFO:tasks.rados.rados.0.smithi033.stdout:2799: done (1 left) 2024-09-21T10:00:15.592 INFO:tasks.rados.rados.0.smithi033.stdout:2800: done (0 left) 2024-09-21T10:00:15.592 INFO:tasks.rados.rados.0.smithi033.stdout:2801: setattr oid 2 current snap is 256 2024-09-21T10:00:15.593 INFO:tasks.rados.rados.0.smithi033.stdout:2802: copy_from oid 19 from oid 8 current snap is 256 2024-09-21T10:00:15.594 INFO:tasks.rados.rados.0.smithi033.stdout:2803: copy_from oid 43 from oid 12 current snap is 256 2024-09-21T10:00:15.594 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 256 2024-09-21T10:00:15.594 INFO:tasks.rados.rados.0.smithi033.stdout:2804: seq_num 887 ranges {3620864=458752} 2024-09-21T10:00:15.617 INFO:tasks.rados.rados.0.smithi033.stdout:2804: writing smithi03385152-31 from 3620864 to 4079616 tid 1 2024-09-21T10:00:15.617 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 256 2024-09-21T10:00:15.617 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 963 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T10:00:15.617 INFO:tasks.rados.rados.0.smithi033.stdout:2802: finishing copy_from racing read to smithi03385152-19 2024-09-21T10:00:15.617 INFO:tasks.rados.rados.0.smithi033.stdout:2802: finishing copy_from to smithi03385152-19 2024-09-21T10:00:15.617 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1485 (ObjNum 877 snap 255 seq_num 877) dirty exists 2024-09-21T10:00:15.617 INFO:tasks.rados.rados.0.smithi033.stdout:2805: seq_num 888 ranges {0=475136} 2024-09-21T10:00:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:2805: writing smithi03385152-28 from 0 to 475136 tid 1 2024-09-21T10:00:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:2801: done (4 left) 2024-09-21T10:00:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:2802: done (3 left) 2024-09-21T10:00:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:2806: read oid 45 snap -1 2024-09-21T10:00:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:2806: expect deleted 2024-09-21T10:00:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:2807: rmattr oid 35 current snap is 256 2024-09-21T10:00:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:2807: done (4 left) 2024-09-21T10:00:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:2808: rollback oid 26 current snap is 256 2024-09-21T10:00:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 26 to 242 2024-09-21T10:00:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 256 2024-09-21T10:00:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:2809: seq_num 889 ranges {2531328=491520} 2024-09-21T10:00:15.637 INFO:tasks.rados.rados.0.smithi033.stdout:2809: writing smithi03385152-39 from 2531328 to 3022848 tid 1 2024-09-21T10:00:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:2810: read oid 41 snap -1 2024-09-21T10:00:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:2810: expect deleted 2024-09-21T10:00:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 256 2024-09-21T10:00:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:2811: seq_num 890 ranges {8331264=507904} 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2811: writing smithi03385152-14 from 8331264 to 8839168 tid 1 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2812: copy_from oid 3 from oid 30 current snap is 256 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2804: finishing write tid 1 to smithi03385152-31 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2804: finishing write tid 2 to smithi03385152-31 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2808: finishing rollback tid 0 to smithi03385152-26 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2805: finishing write tid 1 to smithi03385152-28 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2805: finishing write tid 2 to smithi03385152-28 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2803: finishing copy_from racing read to smithi03385152-43 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:2804: finishing write tid 3 to smithi03385152-31 2024-09-21T10:00:15.687 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2058 (ObjNum 887 snap 256 seq_num 887) dirty exists 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2804: left oid 31 (ObjNum 887 snap 256 seq_num 887) 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2805: finishing write tid 3 to smithi03385152-28 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 900 (ObjNum 888 snap 256 seq_num 888) dirty exists 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2805: left oid 28 (ObjNum 888 snap 256 seq_num 888) 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2808: finishing rollback tid 1 to smithi03385152-26 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2808: finishing rollback tid 2 to smithi03385152-26 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1504 (ObjNum 739 snap 214 seq_num 739) dirty exists 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2803: finishing copy_from to smithi03385152-43 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1499 (ObjNum 851 snap 251 seq_num 851) dirty exists 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2803: done (8 left) 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2804: done (7 left) 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2805: done (6 left) 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2806: done (5 left) 2024-09-21T10:00:15.688 INFO:tasks.rados.rados.0.smithi033.stdout:2808: done (4 left) 2024-09-21T10:00:15.689 INFO:tasks.rados.rados.0.smithi033.stdout:2813: snap_create 2024-09-21T10:00:15.689 INFO:tasks.rados.rados.0.smithi033.stdout:2809: finishing write tid 1 to smithi03385152-39 2024-09-21T10:00:15.689 INFO:tasks.rados.rados.0.smithi033.stdout:2809: finishing write tid 2 to smithi03385152-39 2024-09-21T10:00:15.689 INFO:tasks.rados.rados.0.smithi033.stdout:2809: finishing write tid 3 to smithi03385152-39 2024-09-21T10:00:15.689 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1049 (ObjNum 889 snap 256 seq_num 889) dirty exists 2024-09-21T10:00:15.689 INFO:tasks.rados.rados.0.smithi033.stdout:2809: left oid 39 (ObjNum 889 snap 256 seq_num 889) 2024-09-21T10:00:15.689 INFO:tasks.rados.rados.0.smithi033.stdout:2811: finishing write tid 1 to smithi03385152-14 2024-09-21T10:00:15.689 INFO:tasks.rados.rados.0.smithi033.stdout:2811: finishing write tid 2 to smithi03385152-14 2024-09-21T10:00:15.690 INFO:tasks.rados.rados.0.smithi033.stdout:2811: finishing write tid 3 to smithi03385152-14 2024-09-21T10:00:15.691 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1644 (ObjNum 890 snap 256 seq_num 890) dirty exists 2024-09-21T10:00:15.691 INFO:tasks.rados.rados.0.smithi033.stdout:2811: left oid 14 (ObjNum 890 snap 256 seq_num 890) 2024-09-21T10:00:15.698 INFO:tasks.rados.rados.0.smithi033.stdout:2812: finishing copy_from racing read to smithi03385152-3 2024-09-21T10:00:15.699 INFO:tasks.rados.rados.0.smithi033.stdout:2812: finishing copy_from to smithi03385152-3 2024-09-21T10:00:15.699 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1770 (ObjNum 811 snap 237 seq_num 811) dirty exists 2024-09-21T10:00:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:15 smithi105 ceph-mon[86649]: pgmap v508: 121 pgs: 121 active+clean; 350 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-09-21T10:00:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:15 smithi105 ceph-mon[86649]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T10:00:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:15 smithi033 ceph-mon[96568]: pgmap v508: 121 pgs: 121 active+clean; 350 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-09-21T10:00:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:15 smithi033 ceph-mon[96568]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T10:00:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:15 smithi033 ceph-mon[98546]: pgmap v508: 121 pgs: 121 active+clean; 350 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.1 MiB/s wr, 24 op/s 2024-09-21T10:00:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:15 smithi033 ceph-mon[98546]: osdmap e586: 8 total, 8 up, 8 in 2024-09-21T10:00:16.530 INFO:tasks.rados.rados.0.smithi033.stdout:2809: done (4 left) 2024-09-21T10:00:16.530 INFO:tasks.rados.rados.0.smithi033.stdout:2810: done (3 left) 2024-09-21T10:00:16.530 INFO:tasks.rados.rados.0.smithi033.stdout:2811: done (2 left) 2024-09-21T10:00:16.530 INFO:tasks.rados.rados.0.smithi033.stdout:2812: done (1 left) 2024-09-21T10:00:16.530 INFO:tasks.rados.rados.0.smithi033.stdout:2813: done (0 left) 2024-09-21T10:00:16.530 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 257 2024-09-21T10:00:16.530 INFO:tasks.rados.rados.0.smithi033.stdout:2814: seq_num 891 ranges {385024=532480} 2024-09-21T10:00:16.535 INFO:tasks.rados.rados.0.smithi033.stdout:2814: writing smithi03385152-42 from 385024 to 917504 tid 1 2024-09-21T10:00:16.535 INFO:tasks.rados.rados.0.smithi033.stdout:2815: read oid 35 snap 248 2024-09-21T10:00:16.535 INFO:tasks.rados.rados.0.smithi033.stdout:2815: expect (ObjNum 811 snap 237 seq_num 811) 2024-09-21T10:00:16.535 INFO:tasks.rados.rados.0.smithi033.stdout:2816: delete oid 31 current snap is 257 2024-09-21T10:00:16.539 INFO:tasks.rados.rados.0.smithi033.stdout:2814: finishing write tid 1 to smithi03385152-42 2024-09-21T10:00:16.540 INFO:tasks.rados.rados.0.smithi033.stdout:2814: finishing write tid 2 to smithi03385152-42 2024-09-21T10:00:16.541 INFO:tasks.rados.rados.0.smithi033.stdout:2814: finishing write tid 3 to smithi03385152-42 2024-09-21T10:00:16.541 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 1487 (ObjNum 891 snap 257 seq_num 891) dirty exists 2024-09-21T10:00:16.541 INFO:tasks.rados.rados.0.smithi033.stdout:2814: left oid 42 (ObjNum 891 snap 257 seq_num 891) 2024-09-21T10:00:16.548 INFO:tasks.rados.rados.0.smithi033.stdout:2814: done (2 left) 2024-09-21T10:00:16.549 INFO:tasks.rados.rados.0.smithi033.stdout:2816: done (1 left) 2024-09-21T10:00:16.549 INFO:tasks.rados.rados.0.smithi033.stdout:2817: rmattr oid 11 current snap is 257 2024-09-21T10:00:16.549 INFO:tasks.rados.rados.0.smithi033.stdout:2817: done (1 left) 2024-09-21T10:00:16.549 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 257 2024-09-21T10:00:16.549 INFO:tasks.rados.rados.0.smithi033.stdout:2815: expect (ObjNum 811 snap 237 seq_num 811) 2024-09-21T10:00:16.592 INFO:tasks.rados.rados.0.smithi033.stdout:2818: seq_num 893 ranges {7905280=565248} 2024-09-21T10:00:16.639 INFO:tasks.rados.rados.0.smithi033.stdout:2818: writing smithi03385152-12 from 7905280 to 8470528 tid 1 2024-09-21T10:00:16.639 INFO:tasks.rados.rados.0.smithi033.stdout:2815: done (1 left) 2024-09-21T10:00:16.639 INFO:tasks.rados.rados.0.smithi033.stdout:2819: rollback oid 49 current snap is 257 2024-09-21T10:00:16.639 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 49 to 228 2024-09-21T10:00:16.639 INFO:tasks.rados.rados.0.smithi033.stdout:2820: setattr oid 36 current snap is 257 2024-09-21T10:00:16.640 INFO:tasks.rados.rados.0.smithi033.stdout:2821: rollback oid 13 current snap is 257 2024-09-21T10:00:16.640 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 13 to 247 2024-09-21T10:00:16.640 INFO:tasks.rados.rados.0.smithi033.stdout:2822: snap_create 2024-09-21T10:00:16.645 INFO:tasks.rados.rados.0.smithi033.stdout:2818: finishing write tid 1 to smithi03385152-12 2024-09-21T10:00:16.645 INFO:tasks.rados.rados.0.smithi033.stdout:2818: finishing write tid 2 to smithi03385152-12 2024-09-21T10:00:16.646 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2061 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T10:00:16.647 INFO:tasks.rados.rados.0.smithi033.stdout:2819: finishing rollback tid 0 to smithi03385152-49 2024-09-21T10:00:16.647 INFO:tasks.rados.rados.0.smithi033.stdout:2821: finishing rollback tid 0 to smithi03385152-13 2024-09-21T10:00:16.647 INFO:tasks.rados.rados.0.smithi033.stdout:2819: finishing rollback tid 1 to smithi03385152-49 2024-09-21T10:00:16.647 INFO:tasks.rados.rados.0.smithi033.stdout:2821: finishing rollback tid 1 to smithi03385152-13 2024-09-21T10:00:16.647 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2064 (ObjNum 289 snap 0 seq_num 0) dirty dne 2024-09-21T10:00:16.647 INFO:tasks.rados.rados.0.smithi033.stdout:2818: finishing write tid 3 to smithi03385152-12 2024-09-21T10:00:16.648 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1506 (ObjNum 893 snap 257 seq_num 893) dirty exists 2024-09-21T10:00:16.648 INFO:tasks.rados.rados.0.smithi033.stdout:2818: left oid 12 (ObjNum 893 snap 257 seq_num 893) 2024-09-21T10:00:16.648 INFO:tasks.rados.rados.0.smithi033.stdout:2819: finishing rollback tid 2 to smithi03385152-49 2024-09-21T10:00:16.648 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1509 (ObjNum 751 snap 218 seq_num 751) dirty exists 2024-09-21T10:00:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:16 smithi105 ceph-mon[86649]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T10:00:16.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:16.780 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:16 smithi105 podman[110989]: 2024-09-21 10:00:16.567520619 +0000 UTC m=+6.968791179 container died b0497afc1e278413e71cd22556581c898dac9b7e86c9592845aa8e4a3eddca3e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T10:00:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:16 smithi033 ceph-mon[98546]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T10:00:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:17.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:16 smithi033 ceph-mon[96568]: osdmap e587: 8 total, 8 up, 8 in 2024-09-21T10:00:17.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:17.280 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 podman[110989]: 2024-09-21 10:00:17.077792714 +0000 UTC m=+7.479063262 container cleanup b0497afc1e278413e71cd22556581c898dac9b7e86c9592845aa8e4a3eddca3e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=quincy-fbd1a6a, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:00:17.280 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 bash[110989]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b 2024-09-21T10:00:17.535 INFO:tasks.rados.rados.0.smithi033.stdout:2818: done (4 left) 2024-09-21T10:00:17.536 INFO:tasks.rados.rados.0.smithi033.stdout:2819: done (3 left) 2024-09-21T10:00:17.536 INFO:tasks.rados.rados.0.smithi033.stdout:2820: done (2 left) 2024-09-21T10:00:17.536 INFO:tasks.rados.rados.0.smithi033.stdout:2821: done (1 left) 2024-09-21T10:00:17.536 INFO:tasks.rados.rados.0.smithi033.stdout:2822: done (0 left) 2024-09-21T10:00:17.536 INFO:tasks.rados.rados.0.smithi033.stdout:2823: rollback oid 14 current snap is 258 2024-09-21T10:00:17.536 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 14 to 256 2024-09-21T10:00:17.536 INFO:tasks.rados.rados.0.smithi033.stdout:2824: snap_create 2024-09-21T10:00:17.543 INFO:tasks.rados.rados.0.smithi033.stdout:2823: finishing rollback tid 0 to smithi03385152-14 2024-09-21T10:00:17.544 INFO:tasks.rados.rados.0.smithi033.stdout:2823: finishing rollback tid 1 to smithi03385152-14 2024-09-21T10:00:17.544 INFO:tasks.rados.rados.0.smithi033.stdout:2823: finishing rollback tid 2 to smithi03385152-14 2024-09-21T10:00:17.544 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1773 (ObjNum 890 snap 256 seq_num 890) dirty exists 2024-09-21T10:00:17.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:17 smithi105 ceph-mon[86649]: pgmap v511: 121 pgs: 121 active+clean; 332 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 352 KiB/s wr, 29 op/s 2024-09-21T10:00:17.716 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:17 smithi105 ceph-mon[86649]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T10:00:17.716 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 podman[111008]: 2024-09-21 10:00:17.343011066 +0000 UTC m=+0.768347898 container remove b0497afc1e278413e71cd22556581c898dac9b7e86c9592845aa8e4a3eddca3e (image=quay.ceph.io/ceph-ci/ceph@sha256:ee510334bec3e52f2758802688c49d04ace3cbf9008a76cda46905f72653dab2, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, RELEASE=quincy-fbd1a6a, ceph=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-21T10:00:17.717 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.b.service: Deactivated successfully. 2024-09-21T10:00:17.717 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 systemd[1]: Stopped Ceph mds.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:00:17.717 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.b.service: Consumed 1.743s CPU time. 2024-09-21T10:00:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:17 smithi033 ceph-mon[96568]: pgmap v511: 121 pgs: 121 active+clean; 332 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 352 KiB/s wr, 29 op/s 2024-09-21T10:00:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:17 smithi033 ceph-mon[96568]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T10:00:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:17 smithi033 ceph-mon[98546]: pgmap v511: 121 pgs: 121 active+clean; 332 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 352 KiB/s wr, 29 op/s 2024-09-21T10:00:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:17 smithi033 ceph-mon[98546]: osdmap e588: 8 total, 8 up, 8 in 2024-09-21T10:00:18.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 systemd[1]: Starting Ceph mds.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:00:18.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 podman[111141]: 2024-09-21 10:00:17.824319689 +0000 UTC m=+0.015667122 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:00:18.030 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:17 smithi105 podman[111141]: 2024-09-21 10:00:17.922185383 +0000 UTC m=+0.113532807 container create 1ceb42a1686b3fa095312abf75740c3f302bc7bc8e3eee755c5960d89e693fd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T10:00:18.521 INFO:tasks.rados.rados.0.smithi033.stdout:2823: done (1 left) 2024-09-21T10:00:18.521 INFO:tasks.rados.rados.0.smithi033.stdout:2824: done (0 left) 2024-09-21T10:00:18.521 INFO:tasks.rados.rados.0.smithi033.stdout:2825: read oid 49 snap -1 2024-09-21T10:00:18.521 INFO:tasks.rados.rados.0.smithi033.stdout:2825: expect (ObjNum 751 snap 218 seq_num 751) 2024-09-21T10:00:18.521 INFO:tasks.rados.rados.0.smithi033.stdout:2826: rmattr oid 6 current snap is 259 2024-09-21T10:00:18.521 INFO:tasks.rados.rados.0.smithi033.stdout:2826: done (1 left) 2024-09-21T10:00:18.521 INFO:tasks.rados.rados.0.smithi033.stdout:2827: rollback oid 28 current snap is 259 2024-09-21T10:00:18.522 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 242 2024-09-21T10:00:18.522 INFO:tasks.rados.rados.0.smithi033.stdout:2828: copy_from oid 34 from oid 35 current snap is 259 2024-09-21T10:00:18.522 INFO:tasks.rados.rados.0.smithi033.stdout:2829: delete oid 27 current snap is 259 2024-09-21T10:00:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:2828: finishing copy_from to smithi03385152-34 2024-09-21T10:00:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:2828: got expected ENOENT (src dne) 2024-09-21T10:00:18.526 INFO:tasks.rados.rados.0.smithi033.stdout:2828: finishing copy_from racing read to smithi03385152-34 2024-09-21T10:00:18.527 INFO:tasks.rados.rados.0.smithi033.stdout:2827: finishing rollback tid 0 to smithi03385152-28 2024-09-21T10:00:18.527 INFO:tasks.rados.rados.0.smithi033.stdout:2827: finishing rollback tid 1 to smithi03385152-28 2024-09-21T10:00:18.527 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 902 (ObjNum 33 snap 0 seq_num 113968080) dirty dne 2024-09-21T10:00:18.529 INFO:tasks.rados.rados.0.smithi033.stdout:2827: done (3 left) 2024-09-21T10:00:18.529 INFO:tasks.rados.rados.0.smithi033.stdout:2828: done (2 left) 2024-09-21T10:00:18.529 INFO:tasks.rados.rados.0.smithi033.stdout:2829: done (1 left) 2024-09-21T10:00:18.529 INFO:tasks.rados.rados.0.smithi033.stdout:2830: read oid 13 snap 257 2024-09-21T10:00:18.529 INFO:tasks.rados.rados.0.smithi033.stdout:2830: expect deleted 2024-09-21T10:00:18.529 INFO:tasks.rados.rados.0.smithi033.stdout:2831: snap_create 2024-09-21T10:00:18.530 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:18 smithi105 podman[111141]: 2024-09-21 10:00:18.132840292 +0000 UTC m=+0.324187726 container init 1ceb42a1686b3fa095312abf75740c3f302bc7bc8e3eee755c5960d89e693fd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-21T10:00:18.530 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:18 smithi105 podman[111141]: 2024-09-21 10:00:18.138204499 +0000 UTC m=+0.329551926 container start 1ceb42a1686b3fa095312abf75740c3f302bc7bc8e3eee755c5960d89e693fd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2) 2024-09-21T10:00:18.530 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:18 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b[111156]: starting mds.b at 2024-09-21T10:00:18.530 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:18 smithi105 bash[111141]: 1ceb42a1686b3fa095312abf75740c3f302bc7bc8e3eee755c5960d89e693fd1 2024-09-21T10:00:18.530 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:00:18 smithi105 systemd[1]: Started Ceph mds.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:00:18.547 INFO:tasks.rados.rados.0.smithi033.stdout:2825: expect (ObjNum 751 snap 218 seq_num 751) 2024-09-21T10:00:18.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:18 smithi105 ceph-mon[86649]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T10:00:18.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:18 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:18.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:18 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:18.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:18 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:18.903 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:18 smithi105 ceph-mon[86649]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T10:00:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[98546]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T10:00:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[98546]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T10:00:18.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[96568]: osdmap e589: 8 total, 8 up, 8 in 2024-09-21T10:00:18.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:18.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:18.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:18.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:18 smithi033 ceph-mon[96568]: osdmap e590: 8 total, 8 up, 8 in 2024-09-21T10:00:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:2825: done (2 left) 2024-09-21T10:00:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:2830: done (1 left) 2024-09-21T10:00:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:2831: done (0 left) 2024-09-21T10:00:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:2832: rollback oid 18 current snap is 260 2024-09-21T10:00:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 259 2024-09-21T10:00:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:2833: copy_from oid 24 from oid 30 current snap is 260 2024-09-21T10:00:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:2834: copy_from oid 10 from oid 22 current snap is 260 2024-09-21T10:00:19.520 INFO:tasks.rados.rados.0.smithi033.stdout:2835: snap_remove snap 256 2024-09-21T10:00:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[98546]: pgmap v514: 121 pgs: 121 active+clean; 332 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 560 KiB/s wr, 14 op/s 2024-09-21T10:00:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[98546]: mds.? [v2:172.21.15.105:6832/2342313412,v1:172.21.15.105:6833/2342313412] up:boot 2024-09-21T10:00:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[98546]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-21T10:00:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T10:00:19.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[98546]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T10:00:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[96568]: pgmap v514: 121 pgs: 121 active+clean; 332 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 560 KiB/s wr, 14 op/s 2024-09-21T10:00:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[96568]: mds.? [v2:172.21.15.105:6832/2342313412,v1:172.21.15.105:6833/2342313412] up:boot 2024-09-21T10:00:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[96568]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-21T10:00:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T10:00:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:19 smithi033 ceph-mon[96568]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T10:00:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:19 smithi105 ceph-mon[86649]: pgmap v514: 121 pgs: 121 active+clean; 332 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 560 KiB/s wr, 14 op/s 2024-09-21T10:00:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:19 smithi105 ceph-mon[86649]: mds.? [v2:172.21.15.105:6832/2342313412,v1:172.21.15.105:6833/2342313412] up:boot 2024-09-21T10:00:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:19 smithi105 ceph-mon[86649]: fsmap cephfs:1 {0=a=up:active} 1 up:standby 2024-09-21T10:00:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:19 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "b"} : dispatch 2024-09-21T10:00:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:19 smithi105 ceph-mon[86649]: osdmap e591: 8 total, 8 up, 8 in 2024-09-21T10:00:20.521 INFO:tasks.rados.rados.0.smithi033.stdout:2835: done (3 left) 2024-09-21T10:00:20.521 INFO:tasks.rados.rados.0.smithi033.stdout:2836: read oid 16 snap -1 2024-09-21T10:00:20.521 INFO:tasks.rados.rados.0.smithi033.stdout:2836: expect (ObjNum 862 snap 253 seq_num 862) 2024-09-21T10:00:20.521 INFO:tasks.rados.rados.0.smithi033.stdout:2837: rollback oid 11 current snap is 260 2024-09-21T10:00:20.521 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 11 to 248 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2832: finishing rollback tid 0 to smithi03385152-18 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2832: finishing rollback tid 1 to smithi03385152-18 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2832: finishing rollback tid 2 to smithi03385152-18 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1012 (ObjNum 884 snap 256 seq_num 884) dirty exists 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2833: finishing copy_from racing read to smithi03385152-24 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2833: finishing copy_from to smithi03385152-24 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 1770 (ObjNum 811 snap 237 seq_num 811) dirty exists 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2834: finishing copy_from racing read to smithi03385152-10 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2834: finishing copy_from to smithi03385152-10 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1499 (ObjNum 851 snap 251 seq_num 851) dirty exists 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2832: done (4 left) 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2833: done (3 left) 2024-09-21T10:00:20.522 INFO:tasks.rados.rados.0.smithi033.stdout:2834: done (2 left) 2024-09-21T10:00:20.523 INFO:tasks.rados.rados.0.smithi033.stdout:2838: copy_from oid 45 from oid 9 current snap is 260 2024-09-21T10:00:20.523 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 260 2024-09-21T10:00:20.523 INFO:tasks.rados.rados.0.smithi033.stdout:2839: seq_num 895 ranges {385024=598016} 2024-09-21T10:00:20.527 INFO:tasks.rados.rados.0.smithi033.stdout:2839: writing smithi03385152-26 from 385024 to 983040 tid 1 2024-09-21T10:00:20.527 INFO:tasks.rados.rados.0.smithi033.stdout:2840: snap_remove snap 207 2024-09-21T10:00:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:00:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:00:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:21.536 INFO:tasks.rados.rados.0.smithi033.stdout:2840: done (4 left) 2024-09-21T10:00:21.536 INFO:tasks.rados.rados.0.smithi033.stdout:2841: delete oid 37 current snap is 260 2024-09-21T10:00:21.536 INFO:tasks.rados.rados.0.smithi033.stdout:2836: expect (ObjNum 862 snap 253 seq_num 862) 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2837: finishing rollback tid 1 to smithi03385152-11 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1085 (ObjNum 161 snap 0 seq_num 0) dirty dne 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2838: finishing copy_from racing read to smithi03385152-45 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2838: finishing copy_from to smithi03385152-45 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1483 (ObjNum 860 snap 253 seq_num 860) dirty exists 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2839: finishing write tid 1 to smithi03385152-26 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2839: finishing write tid 2 to smithi03385152-26 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2839: finishing write tid 3 to smithi03385152-26 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1511 (ObjNum 895 snap 260 seq_num 895) dirty exists 2024-09-21T10:00:21.538 INFO:tasks.rados.rados.0.smithi033.stdout:2839: left oid 26 (ObjNum 895 snap 260 seq_num 895) 2024-09-21T10:00:21.548 INFO:tasks.rados.rados.0.smithi033.stdout:2836: done (4 left) 2024-09-21T10:00:21.548 INFO:tasks.rados.rados.0.smithi033.stdout:2837: done (3 left) 2024-09-21T10:00:21.548 INFO:tasks.rados.rados.0.smithi033.stdout:2838: done (2 left) 2024-09-21T10:00:21.548 INFO:tasks.rados.rados.0.smithi033.stdout:2839: done (1 left) 2024-09-21T10:00:21.548 INFO:tasks.rados.rados.0.smithi033.stdout:2841: done (0 left) 2024-09-21T10:00:21.548 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 260 2024-09-21T10:00:21.548 INFO:tasks.rados.rados.0.smithi033.stdout:2842: seq_num 897 ranges {7905280=630784} 2024-09-21T10:00:21.595 INFO:tasks.rados.rados.0.smithi033.stdout:2842: writing smithi03385152-43 from 7905280 to 8536064 tid 1 2024-09-21T10:00:21.595 INFO:tasks.rados.rados.0.smithi033.stdout:2843: snap_remove snap 247 2024-09-21T10:00:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:21 smithi105 ceph-mon[86649]: pgmap v517: 121 pgs: 121 active+clean; 342 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T10:00:21.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:21 smithi105 ceph-mon[86649]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T10:00:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:21 smithi033 ceph-mon[98546]: pgmap v517: 121 pgs: 121 active+clean; 342 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T10:00:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:21 smithi033 ceph-mon[98546]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T10:00:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:21 smithi033 ceph-mon[96568]: pgmap v517: 121 pgs: 121 active+clean; 342 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.7 MiB/s wr, 13 op/s 2024-09-21T10:00:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:21 smithi033 ceph-mon[96568]: osdmap e592: 8 total, 8 up, 8 in 2024-09-21T10:00:21.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:00:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:00:21] "GET /metrics HTTP/1.1" 200 36207 "" "Prometheus/2.51.0" 2024-09-21T10:00:22.543 INFO:tasks.rados.rados.0.smithi033.stdout:2842: finishing write tid 1 to smithi03385152-43 2024-09-21T10:00:22.543 INFO:tasks.rados.rados.0.smithi033.stdout:2842: finishing write tid 2 to smithi03385152-43 2024-09-21T10:00:22.544 INFO:tasks.rados.rados.0.smithi033.stdout:2842: finishing write tid 3 to smithi03385152-43 2024-09-21T10:00:22.544 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1502 (ObjNum 897 snap 260 seq_num 897) dirty exists 2024-09-21T10:00:22.544 INFO:tasks.rados.rados.0.smithi033.stdout:2842: left oid 43 (ObjNum 897 snap 260 seq_num 897) 2024-09-21T10:00:22.544 INFO:tasks.rados.rados.0.smithi033.stdout:2842: done (1 left) 2024-09-21T10:00:22.544 INFO:tasks.rados.rados.0.smithi033.stdout:2843: done (0 left) 2024-09-21T10:00:22.544 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 260 2024-09-21T10:00:22.544 INFO:tasks.rados.rados.0.smithi033.stdout:2844: seq_num 898 ranges {983040=647168} 2024-09-21T10:00:22.553 INFO:tasks.rados.rados.0.smithi033.stdout:2844: writing smithi03385152-26 from 983040 to 1630208 tid 1 2024-09-21T10:00:22.553 INFO:tasks.rados.rados.0.smithi033.stdout:2845: rmattr oid 36 current snap is 260 2024-09-21T10:00:22.554 INFO:tasks.rados.rados.0.smithi033.stdout:2846: snap_remove snap 249 2024-09-21T10:00:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:22 smithi105 ceph-mon[86649]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T10:00:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:22 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:22.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:22 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:22 smithi033 ceph-mon[98546]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T10:00:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:22 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:22 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:22.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:22 smithi033 ceph-mon[96568]: osdmap e593: 8 total, 8 up, 8 in 2024-09-21T10:00:22.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:22 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:22.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:22 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:23.525 INFO:tasks.rados.rados.0.smithi033.stdout:2846: done (2 left) 2024-09-21T10:00:23.526 INFO:tasks.rados.rados.0.smithi033.stdout:2847: snap_remove snap 259 2024-09-21T10:00:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:23 smithi105 ceph-mon[86649]: pgmap v520: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 344 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-09-21T10:00:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:23 smithi105 ceph-mon[86649]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T10:00:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:23 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:23 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:23.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:23 smithi105 ceph-mon[86649]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T10:00:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[98546]: pgmap v520: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 344 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-09-21T10:00:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[98546]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T10:00:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[98546]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T10:00:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[96568]: pgmap v520: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 344 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-09-21T10:00:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[96568]: osdmap e594: 8 total, 8 up, 8 in 2024-09-21T10:00:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:23 smithi033 ceph-mon[96568]: osdmap e595: 8 total, 8 up, 8 in 2024-09-21T10:00:24.526 INFO:tasks.rados.rados.0.smithi033.stdout:2847: done (2 left) 2024-09-21T10:00:24.526 INFO:tasks.rados.rados.0.smithi033.stdout:2848: rmattr oid 29 current snap is 260 2024-09-21T10:00:24.526 INFO:tasks.rados.rados.0.smithi033.stdout:2848: done (2 left) 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2849: read oid 11 snap -1 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2849: expect deleted 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2850: read oid 41 snap -1 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2850: expect deleted 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 260 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2844: finishing write tid 1 to smithi03385152-26 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2844: finishing write tid 2 to smithi03385152-26 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2844: finishing write tid 3 to smithi03385152-26 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1513 (ObjNum 898 snap 260 seq_num 898) dirty exists 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2844: left oid 26 (ObjNum 898 snap 260 seq_num 898) 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2090 (ObjNum 33 snap 0 seq_num 1497754556) dirty exists 2024-09-21T10:00:24.527 INFO:tasks.rados.rados.0.smithi033.stdout:2851: seq_num 899 ranges {409600=663552} 2024-09-21T10:00:24.533 INFO:tasks.rados.rados.0.smithi033.stdout:2851: writing smithi03385152-18 from 409600 to 1073152 tid 1 2024-09-21T10:00:24.534 INFO:tasks.rados.rados.0.smithi033.stdout:2844: done (4 left) 2024-09-21T10:00:24.534 INFO:tasks.rados.rados.0.smithi033.stdout:2845: done (3 left) 2024-09-21T10:00:24.534 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 260 2024-09-21T10:00:24.534 INFO:tasks.rados.rados.0.smithi033.stdout:2852: seq_num 900 ranges {6217728=679936} 2024-09-21T10:00:24.571 INFO:tasks.rados.rados.0.smithi033.stdout:2852: writing smithi03385152-32 from 6217728 to 6897664 tid 1 2024-09-21T10:00:24.572 INFO:tasks.rados.rados.0.smithi033.stdout:2853: snap_remove snap 257 2024-09-21T10:00:25.548 INFO:tasks.rados.rados.0.smithi033.stdout:2849: done (4 left) 2024-09-21T10:00:25.548 INFO:tasks.rados.rados.0.smithi033.stdout:2850: done (3 left) 2024-09-21T10:00:25.548 INFO:tasks.rados.rados.0.smithi033.stdout:2853: done (2 left) 2024-09-21T10:00:25.548 INFO:tasks.rados.rados.0.smithi033.stdout:2854: copy_from oid 4 from oid 27 current snap is 260 2024-09-21T10:00:25.548 INFO:tasks.rados.rados.0.smithi033.stdout:2851: finishing write tid 1 to smithi03385152-18 2024-09-21T10:00:25.548 INFO:tasks.rados.rados.0.smithi033.stdout:2851: finishing write tid 2 to smithi03385152-18 2024-09-21T10:00:25.548 INFO:tasks.rados.rados.0.smithi033.stdout:2851: finishing write tid 3 to smithi03385152-18 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1014 (ObjNum 899 snap 260 seq_num 899) dirty exists 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:2851: left oid 18 (ObjNum 899 snap 260 seq_num 899) 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:2852: finishing write tid 1 to smithi03385152-32 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:2852: finishing write tid 2 to smithi03385152-32 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:2852: finishing write tid 3 to smithi03385152-32 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 292 (ObjNum 900 snap 260 seq_num 900) dirty exists 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:2852: left oid 32 (ObjNum 900 snap 260 seq_num 900) 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:2851: done (2 left) 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:2852: done (1 left) 2024-09-21T10:00:25.549 INFO:tasks.rados.rados.0.smithi033.stdout:2855: snap_remove snap 228 2024-09-21T10:00:25.689 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T10:00:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:25 smithi105 ceph-mon[86649]: pgmap v523: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 317 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 14 op/s 2024-09-21T10:00:25.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:25 smithi105 ceph-mon[86649]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T10:00:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:25 smithi033 ceph-mon[98546]: pgmap v523: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 317 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 14 op/s 2024-09-21T10:00:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:25 smithi033 ceph-mon[98546]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T10:00:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:25 smithi033 ceph-mon[96568]: pgmap v523: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 317 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.2 MiB/s wr, 14 op/s 2024-09-21T10:00:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:25 smithi033 ceph-mon[96568]: osdmap e596: 8 total, 8 up, 8 in 2024-09-21T10:00:26.304 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:00:26.304 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (7m) 23s ago 11m 19.6M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T10:00:26.304 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (6m) 4s ago 11m 88.0M - 9.4.7 954c08fa6188 6f501c82dfe3 2024-09-21T10:00:26.304 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (27s) 23s ago 13m 11.9M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9bc95e6ef3d0 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (8s) 4s ago 13m 14.0M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1ceb42a1686b 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (6m) 4s ago 16m 456M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (6m) 23s ago 18m 528M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (6m) 23s ago 18m 67.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (5m) 4s ago 17m 55.7M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (5m) 23s ago 16m 57.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (7m) 23s ago 11m 12.0M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (6m) 4s ago 11m 12.1M - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (4m) 23s ago 16m 283M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (4m) 23s ago 15m 240M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (3m) 23s ago 15m 154M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 76dbf29565ba 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (2m) 23s ago 15m 83.1M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b85b61817088 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (2m) 4s ago 14m 168M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 01f4f66dbe72 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (112s) 4s ago 14m 207M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4ff1257a8be1 2024-09-21T10:00:26.305 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (78s) 4s ago 14m 102M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 132bc834f24d 2024-09-21T10:00:26.306 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (43s) 4s ago 13m 89.9M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 ea10ccd8e9ee 2024-09-21T10:00:26.306 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (6m) 4s ago 12m 57.4M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T10:00:26.554 INFO:tasks.rados.rados.0.smithi033.stdout:2854: finishing copy_from to smithi03385152-4 2024-09-21T10:00:26.554 INFO:tasks.rados.rados.0.smithi033.stdout:2854: got expected ENOENT (src dne) 2024-09-21T10:00:26.554 INFO:tasks.rados.rados.0.smithi033.stdout:2854: finishing copy_from racing read to smithi03385152-4 2024-09-21T10:00:26.554 INFO:tasks.rados.rados.0.smithi033.stdout:2854: done (1 left) 2024-09-21T10:00:26.554 INFO:tasks.rados.rados.0.smithi033.stdout:2855: done (0 left) 2024-09-21T10:00:26.554 INFO:tasks.rados.rados.0.smithi033.stdout:2856: copy_from oid 3 from oid 2 current snap is 260 2024-09-21T10:00:26.555 INFO:tasks.rados.rados.0.smithi033.stdout:2857: read oid 17 snap -1 2024-09-21T10:00:26.555 INFO:tasks.rados.rados.0.smithi033.stdout:2857: expect deleted 2024-09-21T10:00:26.555 INFO:tasks.rados.rados.0.smithi033.stdout:2858: delete oid 35 current snap is 260 2024-09-21T10:00:26.567 INFO:tasks.rados.rados.0.smithi033.stdout:2857: done (2 left) 2024-09-21T10:00:26.568 INFO:tasks.rados.rados.0.smithi033.stdout:2858: done (1 left) 2024-09-21T10:00:26.568 INFO:tasks.rados.rados.0.smithi033.stdout:2859: setattr oid 27 current snap is 260 2024-09-21T10:00:26.570 INFO:tasks.rados.rados.0.smithi033.stdout:2860: snap_create 2024-09-21T10:00:26.573 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1612 (ObjNum 33 snap 0 seq_num 90725344) dirty exists 2024-09-21T10:00:26.574 INFO:tasks.rados.rados.0.smithi033.stdout:2856: finishing copy_from racing read to smithi03385152-3 2024-09-21T10:00:26.575 INFO:tasks.rados.rados.0.smithi033.stdout:2856: finishing copy_from to smithi03385152-3 2024-09-21T10:00:26.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 963 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T10:00:26.671 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T10:00:26.671 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 15 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T10:00:26.672 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T10:00:26.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T10:00:26.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:26.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:26.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:26.673 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.674 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.675 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.676 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.677 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.678 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:26 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: osdmap e597: 8 total, 8 up, 8 in 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='client.34533 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.a"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.a"}]': finished 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds.b"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds.b"}]': finished 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.782 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:26.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.783 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:26 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:26.970 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T10:00:26.970 INFO:teuthology.orchestra.run.smithi033.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47", 2024-09-21T10:00:26.970 INFO:teuthology.orchestra.run.smithi033.stdout: "in_progress": true, 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "which": "Upgrading all daemon types on all hosts", 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "services_complete": [ 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "osd", 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "mon", 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "mds", 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr" 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: ], 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "progress": "15/20 daemons upgraded", 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "message": "Currently upgrading grafana daemons", 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout: "is_paused": false 2024-09-21T10:00:26.971 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T10:00:27.565 INFO:tasks.rados.rados.0.smithi033.stdout:2856: done (2 left) 2024-09-21T10:00:27.565 INFO:tasks.rados.rados.0.smithi033.stdout:2859: done (1 left) 2024-09-21T10:00:27.565 INFO:tasks.rados.rados.0.smithi033.stdout:2860: done (0 left) 2024-09-21T10:00:27.565 INFO:tasks.rados.rados.0.smithi033.stdout:2861: snap_remove snap 248 2024-09-21T10:00:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: Detected new or changed devices on smithi105 2024-09-21T10:00:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all mds 2024-09-21T10:00:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all rgw 2024-09-21T10:00:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T10:00:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T10:00:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all iscsi 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all nfs 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: pgmap v526: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 255 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/102611701' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[96568]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: Detected new or changed devices on smithi105 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all mds 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all rgw 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T10:00:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all iscsi 2024-09-21T10:00:27.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all nfs 2024-09-21T10:00:27.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:00:27.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:27.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: pgmap v526: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 255 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-21T10:00:27.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T10:00:27.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/102611701' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:27.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:27 smithi033 ceph-mon[98546]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: from='client.44453 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: Detected new or changed devices on smithi105 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all mds 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all rgw 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all rbd-mirror 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all ceph-exporter 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all iscsi 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all nfs 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: Upgrade: Setting container_image for all nvmeof 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: from='client.44459 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: pgmap v526: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 255 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 767 B/s rd, 2.7 MiB/s wr, 5 op/s 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: osdmap e598: 8 total, 8 up, 8 in 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/102611701' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:28.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:27 smithi105 ceph-mon[86649]: from='client.44471 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:28.524 INFO:tasks.rados.rados.0.smithi033.stdout:2861: done (0 left) 2024-09-21T10:00:28.524 INFO:tasks.rados.rados.0.smithi033.stdout:2862: snap_remove snap 239 2024-09-21T10:00:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:28 smithi033 ceph-mon[98546]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T10:00:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:28 smithi033 ceph-mon[98546]: Upgrade: Updating grafana.a 2024-09-21T10:00:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:28 smithi033 ceph-mon[98546]: Deploying daemon grafana.a on smithi105 2024-09-21T10:00:28.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:28 smithi033 ceph-mon[98546]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T10:00:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:28 smithi033 ceph-mon[96568]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T10:00:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:28 smithi033 ceph-mon[96568]: Upgrade: Updating grafana.a 2024-09-21T10:00:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:28 smithi033 ceph-mon[96568]: Deploying daemon grafana.a on smithi105 2024-09-21T10:00:28.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:28 smithi033 ceph-mon[96568]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T10:00:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:28 smithi105 ceph-mon[86649]: osdmap e599: 8 total, 8 up, 8 in 2024-09-21T10:00:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:28 smithi105 ceph-mon[86649]: Upgrade: Updating grafana.a 2024-09-21T10:00:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:28 smithi105 ceph-mon[86649]: Deploying daemon grafana.a on smithi105 2024-09-21T10:00:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:28 smithi105 ceph-mon[86649]: osdmap e600: 8 total, 8 up, 8 in 2024-09-21T10:00:29.527 INFO:tasks.rados.rados.0.smithi033.stdout:2862: done (0 left) 2024-09-21T10:00:29.527 INFO:tasks.rados.rados.0.smithi033.stdout:2863: rollback oid 18 current snap is 261 2024-09-21T10:00:29.527 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 252 2024-09-21T10:00:29.527 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 261 2024-09-21T10:00:29.527 INFO:tasks.rados.rados.0.smithi033.stdout:2864: seq_num 902 ranges {7905280=712704} 2024-09-21T10:00:29.576 INFO:tasks.rados.rados.0.smithi033.stdout:2864: writing smithi03385152-10 from 7905280 to 8617984 tid 1 2024-09-21T10:00:29.576 INFO:tasks.rados.rados.0.smithi033.stdout:2865: snap_create 2024-09-21T10:00:29.576 INFO:tasks.rados.rados.0.smithi033.stdout:2863: finishing rollback tid 0 to smithi03385152-18 2024-09-21T10:00:29.576 INFO:tasks.rados.rados.0.smithi033.stdout:2863: finishing rollback tid 1 to smithi03385152-18 2024-09-21T10:00:29.576 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1016 (ObjNum 289 snap 0 seq_num 91536880) dirty dne 2024-09-21T10:00:29.580 INFO:tasks.rados.rados.0.smithi033.stdout:2864: finishing write tid 1 to smithi03385152-10 2024-09-21T10:00:29.580 INFO:tasks.rados.rados.0.smithi033.stdout:2864: finishing write tid 2 to smithi03385152-10 2024-09-21T10:00:29.582 INFO:tasks.rados.rados.0.smithi033.stdout:2864: finishing write tid 3 to smithi03385152-10 2024-09-21T10:00:29.582 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1937 (ObjNum 902 snap 261 seq_num 902) dirty exists 2024-09-21T10:00:29.582 INFO:tasks.rados.rados.0.smithi033.stdout:2864: left oid 10 (ObjNum 902 snap 261 seq_num 902) 2024-09-21T10:00:29.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:29 smithi105 ceph-mon[86649]: pgmap v529: 121 pgs: 121 active+clean; 255 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 316 KiB/s wr, 2 op/s 2024-09-21T10:00:29.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:29 smithi105 ceph-mon[86649]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T10:00:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:29 smithi033 ceph-mon[98546]: pgmap v529: 121 pgs: 121 active+clean; 255 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 316 KiB/s wr, 2 op/s 2024-09-21T10:00:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:29 smithi033 ceph-mon[98546]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T10:00:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:29 smithi033 ceph-mon[96568]: pgmap v529: 121 pgs: 121 active+clean; 255 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 316 KiB/s wr, 2 op/s 2024-09-21T10:00:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:29 smithi033 ceph-mon[96568]: osdmap e601: 8 total, 8 up, 8 in 2024-09-21T10:00:30.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:00:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:00:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:30.581 INFO:tasks.rados.rados.0.smithi033.stdout:2863: done (2 left) 2024-09-21T10:00:30.581 INFO:tasks.rados.rados.0.smithi033.stdout:2864: done (1 left) 2024-09-21T10:00:30.582 INFO:tasks.rados.rados.0.smithi033.stdout:2865: done (0 left) 2024-09-21T10:00:30.582 INFO:tasks.rados.rados.0.smithi033.stdout:2866: rollback oid 20 current snap is 262 2024-09-21T10:00:30.582 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 242 2024-09-21T10:00:30.582 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 262 2024-09-21T10:00:30.582 INFO:tasks.rados.rados.0.smithi033.stdout:2867: seq_num 903 ranges {0=729088} 2024-09-21T10:00:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:2867: writing smithi03385152-35 from 0 to 729088 tid 1 2024-09-21T10:00:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:2868: delete oid 5 current snap is 262 2024-09-21T10:00:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:2866: finishing rollback tid 0 to smithi03385152-20 2024-09-21T10:00:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:2866: finishing rollback tid 1 to smithi03385152-20 2024-09-21T10:00:30.586 INFO:tasks.rados.rados.0.smithi033.stdout:2866: finishing rollback tid 2 to smithi03385152-20 2024-09-21T10:00:30.586 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1513 (ObjNum 807 snap 234 seq_num 807) dirty exists 2024-09-21T10:00:30.591 INFO:tasks.rados.rados.0.smithi033.stdout:2866: done (2 left) 2024-09-21T10:00:30.591 INFO:tasks.rados.rados.0.smithi033.stdout:2868: done (1 left) 2024-09-21T10:00:30.591 INFO:tasks.rados.rados.0.smithi033.stdout:2869: setattr oid 3 current snap is 262 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2867: finishing write tid 1 to smithi03385152-35 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2870: rmattr oid 7 current snap is 262 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2871: rmattr oid 1 current snap is 262 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2871: done (3 left) 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2872: read oid 19 snap 261 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2872: expect (ObjNum 877 snap 255 seq_num 877) 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2873: read oid 45 snap -1 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2873: expect (ObjNum 860 snap 253 seq_num 860) 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2874: delete oid 39 current snap is 262 2024-09-21T10:00:30.592 INFO:tasks.rados.rados.0.smithi033.stdout:2867: finishing write tid 2 to smithi03385152-35 2024-09-21T10:00:30.595 INFO:tasks.rados.rados.0.smithi033.stdout:2874: done (5 left) 2024-09-21T10:00:30.595 INFO:tasks.rados.rados.0.smithi033.stdout:2875: snap_create 2024-09-21T10:00:30.596 INFO:tasks.rados.rados.0.smithi033.stdout:2872: expect (ObjNum 877 snap 255 seq_num 877) 2024-09-21T10:00:30.621 INFO:tasks.rados.rados.0.smithi033.stdout:2867: finishing write tid 3 to smithi03385152-35 2024-09-21T10:00:30.622 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2135 (ObjNum 903 snap 262 seq_num 903) dirty exists 2024-09-21T10:00:30.622 INFO:tasks.rados.rados.0.smithi033.stdout:2867: left oid 35 (ObjNum 903 snap 262 seq_num 903) 2024-09-21T10:00:30.622 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1774 (ObjNum 747 snap 216 seq_num 747) dirty exists 2024-09-21T10:00:30.622 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2137 (ObjNum 724 snap 205 seq_num 724) dirty exists 2024-09-21T10:00:30.622 INFO:tasks.rados.rados.0.smithi033.stdout:2873: expect (ObjNum 860 snap 253 seq_num 860) 2024-09-21T10:00:31.591 INFO:tasks.rados.rados.0.smithi033.stdout:2867: done (5 left) 2024-09-21T10:00:31.591 INFO:tasks.rados.rados.0.smithi033.stdout:2869: done (4 left) 2024-09-21T10:00:31.591 INFO:tasks.rados.rados.0.smithi033.stdout:2870: done (3 left) 2024-09-21T10:00:31.591 INFO:tasks.rados.rados.0.smithi033.stdout:2872: done (2 left) 2024-09-21T10:00:31.591 INFO:tasks.rados.rados.0.smithi033.stdout:2873: done (1 left) 2024-09-21T10:00:31.591 INFO:tasks.rados.rados.0.smithi033.stdout:2875: done (0 left) 2024-09-21T10:00:31.591 INFO:tasks.rados.rados.0.smithi033.stdout:2876: snap_remove snap 262 2024-09-21T10:00:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:31 smithi105 ceph-mon[86649]: pgmap v532: 121 pgs: 121 active+clean; 236 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 328 KiB/s wr, 6 op/s 2024-09-21T10:00:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:31 smithi105 ceph-mon[86649]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T10:00:31.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:00:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:00:31] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-09-21T10:00:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:31 smithi033 ceph-mon[98546]: pgmap v532: 121 pgs: 121 active+clean; 236 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 328 KiB/s wr, 6 op/s 2024-09-21T10:00:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:31 smithi033 ceph-mon[98546]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T10:00:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:31 smithi033 ceph-mon[96568]: pgmap v532: 121 pgs: 121 active+clean; 236 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 328 KiB/s wr, 6 op/s 2024-09-21T10:00:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:31 smithi033 ceph-mon[96568]: osdmap e602: 8 total, 8 up, 8 in 2024-09-21T10:00:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:32.596 INFO:tasks.rados.rados.0.smithi033.stdout:2876: done (0 left) 2024-09-21T10:00:32.597 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 263 2024-09-21T10:00:32.597 INFO:tasks.rados.rados.0.smithi033.stdout:2877: seq_num 906 ranges {0=778240} 2024-09-21T10:00:32.601 INFO:tasks.rados.rados.0.smithi033.stdout:2877: writing smithi03385152-31 from 0 to 778240 tid 1 2024-09-21T10:00:32.601 INFO:tasks.rados.rados.0.smithi033.stdout:2878: delete oid 49 current snap is 263 2024-09-21T10:00:32.606 INFO:tasks.rados.rados.0.smithi033.stdout:2878: done (1 left) 2024-09-21T10:00:32.607 INFO:tasks.rados.rados.0.smithi033.stdout:2879: delete oid 34 current snap is 263 2024-09-21T10:00:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:2877: finishing write tid 1 to smithi03385152-31 2024-09-21T10:00:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:2877: finishing write tid 2 to smithi03385152-31 2024-09-21T10:00:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:2879: done (1 left) 2024-09-21T10:00:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:2880: snap_remove snap 258 2024-09-21T10:00:32.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:32 smithi105 ceph-mon[86649]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T10:00:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:32 smithi033 ceph-mon[98546]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T10:00:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:32 smithi033 ceph-mon[96568]: osdmap e603: 8 total, 8 up, 8 in 2024-09-21T10:00:33.528 INFO:tasks.rados.rados.0.smithi033.stdout:2880: done (1 left) 2024-09-21T10:00:33.528 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 263 2024-09-21T10:00:33.528 INFO:tasks.rados.rados.0.smithi033.stdout:2881: seq_num 909 ranges {8536064=425984,8962048=401408} 2024-09-21T10:00:33.577 INFO:tasks.rados.rados.0.smithi033.stdout:2881: writing smithi03385152-43 from 8536064 to 8962048 tid 1 2024-09-21T10:00:33.580 INFO:tasks.rados.rados.0.smithi033.stdout:2881: writing smithi03385152-43 from 8962048 to 9363456 tid 2 2024-09-21T10:00:33.580 INFO:tasks.rados.rados.0.smithi033.stdout:2882: read oid 22 snap -1 2024-09-21T10:00:33.580 INFO:tasks.rados.rados.0.smithi033.stdout:2882: expect (ObjNum 851 snap 251 seq_num 851) 2024-09-21T10:00:33.580 INFO:tasks.rados.rados.0.smithi033.stdout:2883: read oid 12 snap -1 2024-09-21T10:00:33.580 INFO:tasks.rados.rados.0.smithi033.stdout:2883: expect (ObjNum 893 snap 257 seq_num 893) 2024-09-21T10:00:33.580 INFO:tasks.rados.rados.0.smithi033.stdout:2884: snap_remove snap 252 2024-09-21T10:00:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:33 smithi033 ceph-mon[98546]: pgmap v535: 121 pgs: 121 active+clean; 207 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 470 KiB/s wr, 14 op/s 2024-09-21T10:00:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:33 smithi033 ceph-mon[98546]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T10:00:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:33 smithi033 ceph-mon[98546]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T10:00:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:33 smithi033 ceph-mon[96568]: pgmap v535: 121 pgs: 121 active+clean; 207 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 470 KiB/s wr, 14 op/s 2024-09-21T10:00:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:33 smithi033 ceph-mon[96568]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T10:00:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:33 smithi033 ceph-mon[96568]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T10:00:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:33 smithi105 ceph-mon[86649]: pgmap v535: 121 pgs: 121 active+clean; 207 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.7 KiB/s rd, 470 KiB/s wr, 14 op/s 2024-09-21T10:00:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:33 smithi105 ceph-mon[86649]: osdmap e604: 8 total, 8 up, 8 in 2024-09-21T10:00:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:33 smithi105 ceph-mon[86649]: osdmap e605: 8 total, 8 up, 8 in 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:2877: finishing write tid 3 to smithi03385152-31 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2139 (ObjNum 906 snap 263 seq_num 906) dirty exists 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:2877: left oid 31 (ObjNum 906 snap 263 seq_num 906) 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:2881: finishing write tid 1 to smithi03385152-43 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:2881: finishing write tid 2 to smithi03385152-43 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:2881: finishing write tid 3 to smithi03385152-43 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:2881: finishing write tid 4 to smithi03385152-43 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1505 (ObjNum 909 snap 263 seq_num 909) dirty exists 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:2881: left oid 43 (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:00:34.529 INFO:tasks.rados.rados.0.smithi033.stdout:2882: expect (ObjNum 851 snap 251 seq_num 851) 2024-09-21T10:00:35.020 INFO:tasks.rados.rados.0.smithi033.stdout:2883: expect (ObjNum 893 snap 257 seq_num 893) 2024-09-21T10:00:35.644 INFO:tasks.rados.rados.0.smithi033.stdout:2877: done (4 left) 2024-09-21T10:00:35.644 INFO:tasks.rados.rados.0.smithi033.stdout:2881: done (3 left) 2024-09-21T10:00:35.644 INFO:tasks.rados.rados.0.smithi033.stdout:2882: done (2 left) 2024-09-21T10:00:35.644 INFO:tasks.rados.rados.0.smithi033.stdout:2883: done (1 left) 2024-09-21T10:00:35.644 INFO:tasks.rados.rados.0.smithi033.stdout:2884: done (0 left) 2024-09-21T10:00:35.644 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 263 2024-09-21T10:00:35.644 INFO:tasks.rados.rados.0.smithi033.stdout:2885: seq_num 910 ranges {729088=442368,1171456=401408} 2024-09-21T10:00:35.650 INFO:tasks.rados.rados.0.smithi033.stdout:2885: writing smithi03385152-35 from 729088 to 1171456 tid 1 2024-09-21T10:00:35.652 INFO:tasks.rados.rados.0.smithi033.stdout:2885: writing smithi03385152-35 from 1171456 to 1572864 tid 2 2024-09-21T10:00:35.652 INFO:tasks.rados.rados.0.smithi033.stdout:2886: snap_create 2024-09-21T10:00:35.655 INFO:tasks.rados.rados.0.smithi033.stdout:2885: finishing write tid 1 to smithi03385152-35 2024-09-21T10:00:35.655 INFO:tasks.rados.rados.0.smithi033.stdout:2885: finishing write tid 2 to smithi03385152-35 2024-09-21T10:00:35.655 INFO:tasks.rados.rados.0.smithi033.stdout:2885: finishing write tid 3 to smithi03385152-35 2024-09-21T10:00:35.656 INFO:tasks.rados.rados.0.smithi033.stdout:2885: finishing write tid 4 to smithi03385152-35 2024-09-21T10:00:35.656 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2156 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:00:35.657 INFO:tasks.rados.rados.0.smithi033.stdout:2885: left oid 35 (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:00:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:35 smithi105 ceph-mon[86649]: pgmap v538: 121 pgs: 121 active+clean; 208 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 478 KiB/s wr, 12 op/s 2024-09-21T10:00:35.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:35 smithi105 ceph-mon[86649]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T10:00:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:35 smithi033 ceph-mon[98546]: pgmap v538: 121 pgs: 121 active+clean; 208 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 478 KiB/s wr, 12 op/s 2024-09-21T10:00:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:35 smithi033 ceph-mon[98546]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T10:00:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:35 smithi033 ceph-mon[96568]: pgmap v538: 121 pgs: 121 active+clean; 208 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 478 KiB/s wr, 12 op/s 2024-09-21T10:00:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:35 smithi033 ceph-mon[96568]: osdmap e606: 8 total, 8 up, 8 in 2024-09-21T10:00:36.535 INFO:tasks.rados.rados.0.smithi033.stdout:2885: done (1 left) 2024-09-21T10:00:36.535 INFO:tasks.rados.rados.0.smithi033.stdout:2886: done (0 left) 2024-09-21T10:00:36.535 INFO:tasks.rados.rados.0.smithi033.stdout:2887: read oid 32 snap -1 2024-09-21T10:00:36.535 INFO:tasks.rados.rados.0.smithi033.stdout:2887: expect (ObjNum 900 snap 260 seq_num 900) 2024-09-21T10:00:36.535 INFO:tasks.rados.rados.0.smithi033.stdout:2888: delete oid 47 current snap is 264 2024-09-21T10:00:36.549 INFO:tasks.rados.rados.0.smithi033.stdout:2888: done (1 left) 2024-09-21T10:00:36.549 INFO:tasks.rados.rados.0.smithi033.stdout:2889: setattr oid 28 current snap is 264 2024-09-21T10:00:36.553 INFO:tasks.rados.rados.0.smithi033.stdout:2890: read oid 30 snap 263 2024-09-21T10:00:36.553 INFO:tasks.rados.rados.0.smithi033.stdout:2890: expect (ObjNum 811 snap 237 seq_num 811) 2024-09-21T10:00:36.553 INFO:tasks.rados.rados.0.smithi033.stdout:2891: copy_from oid 19 from oid 37 current snap is 264 2024-09-21T10:00:36.553 INFO:tasks.rados.rados.0.smithi033.stdout:2892: read oid 4 snap -1 2024-09-21T10:00:36.553 INFO:tasks.rados.rados.0.smithi033.stdout:2892: expect (ObjNum 746 snap 216 seq_num 746) 2024-09-21T10:00:36.553 INFO:tasks.rados.rados.0.smithi033.stdout:2893: setattr oid 41 current snap is 264 2024-09-21T10:00:36.557 INFO:tasks.rados.rados.0.smithi033.stdout:2894: rmattr oid 18 current snap is 264 2024-09-21T10:00:36.557 INFO:tasks.rados.rados.0.smithi033.stdout:2894: done (6 left) 2024-09-21T10:00:36.557 INFO:tasks.rados.rados.0.smithi033.stdout:2895: read oid 3 snap 242 2024-09-21T10:00:36.557 INFO:tasks.rados.rados.0.smithi033.stdout:2895: expect deleted 2024-09-21T10:00:36.558 INFO:tasks.rados.rados.0.smithi033.stdout:2896: snap_create 2024-09-21T10:00:36.558 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 903 (ObjNum 129 snap 0 seq_num 91030304) dirty exists 2024-09-21T10:00:36.560 INFO:tasks.rados.rados.0.smithi033.stdout:2891: finishing copy_from to smithi03385152-19 2024-09-21T10:00:36.560 INFO:tasks.rados.rados.0.smithi033.stdout:2891: got expected ENOENT (src dne) 2024-09-21T10:00:36.560 INFO:tasks.rados.rados.0.smithi033.stdout:2891: finishing copy_from racing read to smithi03385152-19 2024-09-21T10:00:36.566 INFO:tasks.rados.rados.0.smithi033.stdout:2887: expect (ObjNum 900 snap 260 seq_num 900) 2024-09-21T10:00:37.153 INFO:tasks.rados.rados.0.smithi033.stdout:2890: expect (ObjNum 811 snap 237 seq_num 811) 2024-09-21T10:00:37.196 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 1488 (ObjNum 65 snap 0 seq_num 1383093604) dirty exists 2024-09-21T10:00:37.196 INFO:tasks.rados.rados.0.smithi033.stdout:2892: expect (ObjNum 746 snap 216 seq_num 746) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2887: done (7 left) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2889: done (6 left) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2890: done (5 left) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2891: done (4 left) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2892: done (3 left) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2893: done (2 left) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2895: done (1 left) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2896: done (0 left) 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 265 2024-09-21T10:00:37.540 INFO:tasks.rados.rados.0.smithi033.stdout:2897: seq_num 912 ranges {2179072=483328,2662400=401408} 2024-09-21T10:00:37.554 INFO:tasks.rados.rados.0.smithi033.stdout:2897: writing smithi03385152-15 from 2179072 to 2662400 tid 1 2024-09-21T10:00:37.556 INFO:tasks.rados.rados.0.smithi033.stdout:2897: writing smithi03385152-15 from 2662400 to 3063808 tid 2 2024-09-21T10:00:37.557 INFO:tasks.rados.rados.0.smithi033.stdout:2898: delete oid 23 current snap is 265 2024-09-21T10:00:37.559 INFO:tasks.rados.rados.0.smithi033.stdout:2898: done (1 left) 2024-09-21T10:00:37.559 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 265 2024-09-21T10:00:37.559 INFO:tasks.rados.rados.0.smithi033.stdout:2899: seq_num 914 ranges {0=516096,516096=401408} 2024-09-21T10:00:37.561 INFO:tasks.rados.rados.0.smithi033.stdout:2899: writing smithi03385152-28 from 0 to 516096 tid 1 2024-09-21T10:00:37.564 INFO:tasks.rados.rados.0.smithi033.stdout:2899: writing smithi03385152-28 from 516096 to 917504 tid 2 2024-09-21T10:00:37.564 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 265 2024-09-21T10:00:37.564 INFO:tasks.rados.rados.0.smithi033.stdout:2900: seq_num 915 ranges {0=532480,532480=401408} 2024-09-21T10:00:37.567 INFO:tasks.rados.rados.0.smithi033.stdout:2900: writing smithi03385152-37 from 0 to 532480 tid 1 2024-09-21T10:00:37.570 INFO:tasks.rados.rados.0.smithi033.stdout:2900: writing smithi03385152-37 from 532480 to 933888 tid 2 2024-09-21T10:00:37.570 INFO:tasks.rados.rados.0.smithi033.stdout:2901: read oid 42 snap -1 2024-09-21T10:00:37.570 INFO:tasks.rados.rados.0.smithi033.stdout:2901: expect (ObjNum 891 snap 257 seq_num 891) 2024-09-21T10:00:37.570 INFO:tasks.rados.rados.0.smithi033.stdout:2902: delete oid 29 current snap is 265 2024-09-21T10:00:37.570 INFO:tasks.rados.rados.0.smithi033.stdout:2897: finishing write tid 1 to smithi03385152-15 2024-09-21T10:00:37.570 INFO:tasks.rados.rados.0.smithi033.stdout:2897: finishing write tid 2 to smithi03385152-15 2024-09-21T10:00:37.570 INFO:tasks.rados.rados.0.smithi033.stdout:2897: finishing write tid 3 to smithi03385152-15 2024-09-21T10:00:37.570 INFO:tasks.rados.rados.0.smithi033.stdout:2897: finishing write tid 4 to smithi03385152-15 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 1491 (ObjNum 912 snap 265 seq_num 912) dirty exists 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:2897: left oid 15 (ObjNum 912 snap 265 seq_num 912) 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:2899: finishing write tid 1 to smithi03385152-28 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:2899: finishing write tid 2 to smithi03385152-28 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:2899: finishing write tid 3 to smithi03385152-28 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:2899: finishing write tid 4 to smithi03385152-28 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 906 (ObjNum 914 snap 265 seq_num 914) dirty exists 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:2899: left oid 28 (ObjNum 914 snap 265 seq_num 914) 2024-09-21T10:00:37.571 INFO:tasks.rados.rados.0.smithi033.stdout:2900: finishing write tid 1 to smithi03385152-37 2024-09-21T10:00:37.572 INFO:tasks.rados.rados.0.smithi033.stdout:2897: done (4 left) 2024-09-21T10:00:37.572 INFO:tasks.rados.rados.0.smithi033.stdout:2899: done (3 left) 2024-09-21T10:00:37.572 INFO:tasks.rados.rados.0.smithi033.stdout:2902: done (2 left) 2024-09-21T10:00:37.572 INFO:tasks.rados.rados.0.smithi033.stdout:2903: copy_from oid 27 from oid 23 current snap is 265 2024-09-21T10:00:37.573 INFO:tasks.rados.rados.0.smithi033.stdout:2904: copy_from oid 14 from oid 6 current snap is 265 2024-09-21T10:00:37.573 INFO:tasks.rados.rados.0.smithi033.stdout:2905: snap_create 2024-09-21T10:00:37.573 INFO:tasks.rados.rados.0.smithi033.stdout:2900: finishing write tid 2 to smithi03385152-37 2024-09-21T10:00:37.573 INFO:tasks.rados.rados.0.smithi033.stdout:2900: finishing write tid 3 to smithi03385152-37 2024-09-21T10:00:37.573 INFO:tasks.rados.rados.0.smithi033.stdout:2904: finishing copy_from to smithi03385152-14 2024-09-21T10:00:37.573 INFO:tasks.rados.rados.0.smithi033.stdout:2904: got expected ENOENT (src dne) 2024-09-21T10:00:37.574 INFO:tasks.rados.rados.0.smithi033.stdout:2904: finishing copy_from racing read to smithi03385152-14 2024-09-21T10:00:37.574 INFO:tasks.rados.rados.0.smithi033.stdout:2903: finishing copy_from to smithi03385152-27 2024-09-21T10:00:37.574 INFO:tasks.rados.rados.0.smithi033.stdout:2903: got expected ENOENT (src dne) 2024-09-21T10:00:37.574 INFO:tasks.rados.rados.0.smithi033.stdout:2901: expect (ObjNum 891 snap 257 seq_num 891) 2024-09-21T10:00:37.605 INFO:tasks.rados.rados.0.smithi033.stdout:2903: finishing copy_from racing read to smithi03385152-27 2024-09-21T10:00:37.605 INFO:tasks.rados.rados.0.smithi033.stdout:2900: finishing write tid 4 to smithi03385152-37 2024-09-21T10:00:37.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1508 (ObjNum 915 snap 265 seq_num 915) dirty exists 2024-09-21T10:00:37.605 INFO:tasks.rados.rados.0.smithi033.stdout:2900: left oid 37 (ObjNum 915 snap 265 seq_num 915) 2024-09-21T10:00:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:37 smithi105 ceph-mon[86649]: pgmap v540: 121 pgs: 121 active+clean; 200 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 471 KiB/s wr, 6 op/s 2024-09-21T10:00:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:37 smithi105 ceph-mon[86649]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T10:00:37.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:37 smithi033 ceph-mon[98546]: pgmap v540: 121 pgs: 121 active+clean; 200 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 471 KiB/s wr, 6 op/s 2024-09-21T10:00:37.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:37 smithi033 ceph-mon[98546]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T10:00:37.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:37 smithi033 ceph-mon[96568]: pgmap v540: 121 pgs: 121 active+clean; 200 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 471 KiB/s wr, 6 op/s 2024-09-21T10:00:37.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:37 smithi033 ceph-mon[96568]: osdmap e607: 8 total, 8 up, 8 in 2024-09-21T10:00:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2900: done (4 left) 2024-09-21T10:00:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2901: done (3 left) 2024-09-21T10:00:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2903: done (2 left) 2024-09-21T10:00:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2904: done (1 left) 2024-09-21T10:00:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2905: done (0 left) 2024-09-21T10:00:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2906: copy_from oid 4 from oid 16 current snap is 266 2024-09-21T10:00:38.527 INFO:tasks.rados.rados.0.smithi033.stdout:2907: snap_create 2024-09-21T10:00:38.534 INFO:tasks.rados.rados.0.smithi033.stdout:2906: finishing copy_from racing read to smithi03385152-4 2024-09-21T10:00:38.535 INFO:tasks.rados.rados.0.smithi033.stdout:2906: finishing copy_from to smithi03385152-4 2024-09-21T10:00:38.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1964 (ObjNum 862 snap 253 seq_num 862) dirty exists 2024-09-21T10:00:38.714 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:38 smithi105 systemd[1]: Stopping Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:00:38.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:38 smithi105 ceph-mon[86649]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T10:00:38.714 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:38 smithi105 ceph-mon[86649]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T10:00:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:38 smithi033 ceph-mon[98546]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T10:00:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:38 smithi033 ceph-mon[98546]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T10:00:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:38 smithi033 ceph-mon[96568]: osdmap e608: 8 total, 8 up, 8 in 2024-09-21T10:00:38.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:38 smithi033 ceph-mon[96568]: osdmap e609: 8 total, 8 up, 8 in 2024-09-21T10:00:39.010 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:38 smithi105 podman[114996]: 2024-09-21 10:00:38.711718065 +0000 UTC m=+0.130560859 container died 6f501c82dfe359e0f9c3e702828bad29f98f989ba91356456d5366e6adf44733 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vcs-type=git, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, name=ubi8, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5) 2024-09-21T10:00:39.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:39 smithi105 podman[114996]: 2024-09-21 10:00:39.090657584 +0000 UTC m=+0.509500381 container cleanup 6f501c82dfe359e0f9c3e702828bad29f98f989ba91356456d5366e6adf44733 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, description=Ceph Grafana Container, vendor=Red Hat, Inc., io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, io.openshift.expose-services=, version=8.5, build-date=2022-04-29T11:48:11.055897) 2024-09-21T10:00:39.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:39 smithi105 bash[114996]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a 2024-09-21T10:00:39.534 INFO:tasks.rados.rados.0.smithi033.stdout:2906: done (1 left) 2024-09-21T10:00:39.535 INFO:tasks.rados.rados.0.smithi033.stdout:2907: done (0 left) 2024-09-21T10:00:39.535 INFO:tasks.rados.rados.0.smithi033.stdout:2908: read oid 49 snap -1 2024-09-21T10:00:39.535 INFO:tasks.rados.rados.0.smithi033.stdout:2908: expect deleted 2024-09-21T10:00:39.535 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 267 2024-09-21T10:00:39.535 INFO:tasks.rados.rados.0.smithi033.stdout:2909: seq_num 917 ranges {7905280=565248,8470528=401408} 2024-09-21T10:00:39.580 INFO:tasks.rados.rados.0.smithi033.stdout:2909: writing smithi03385152-22 from 7905280 to 8470528 tid 1 2024-09-21T10:00:39.583 INFO:tasks.rados.rados.0.smithi033.stdout:2909: writing smithi03385152-22 from 8470528 to 8871936 tid 2 2024-09-21T10:00:39.583 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 267 2024-09-21T10:00:39.583 INFO:tasks.rados.rados.0.smithi033.stdout:2910: seq_num 918 ranges {1630208=581632,2211840=401408} 2024-09-21T10:00:39.595 INFO:tasks.rados.rados.0.smithi033.stdout:2910: writing smithi03385152-26 from 1630208 to 2211840 tid 1 2024-09-21T10:00:39.598 INFO:tasks.rados.rados.0.smithi033.stdout:2910: writing smithi03385152-26 from 2211840 to 2613248 tid 2 2024-09-21T10:00:39.598 INFO:tasks.rados.rados.0.smithi033.stdout:2911: copy_from oid 45 from oid 28 current snap is 267 2024-09-21T10:00:39.598 INFO:tasks.rados.rados.0.smithi033.stdout:2909: finishing write tid 1 to smithi03385152-22 2024-09-21T10:00:39.598 INFO:tasks.rados.rados.0.smithi033.stdout:2909: finishing write tid 2 to smithi03385152-22 2024-09-21T10:00:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:2909: finishing write tid 3 to smithi03385152-22 2024-09-21T10:00:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:2909: finishing write tid 4 to smithi03385152-22 2024-09-21T10:00:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1778 (ObjNum 917 snap 267 seq_num 917) dirty exists 2024-09-21T10:00:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:2909: left oid 22 (ObjNum 917 snap 267 seq_num 917) 2024-09-21T10:00:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:2908: done (3 left) 2024-09-21T10:00:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:2909: done (2 left) 2024-09-21T10:00:39.599 INFO:tasks.rados.rados.0.smithi033.stdout:2912: snap_create 2024-09-21T10:00:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:2910: finishing write tid 1 to smithi03385152-26 2024-09-21T10:00:39.600 INFO:tasks.rados.rados.0.smithi033.stdout:2910: finishing write tid 2 to smithi03385152-26 2024-09-21T10:00:39.601 INFO:tasks.rados.rados.0.smithi033.stdout:2910: finishing write tid 3 to smithi03385152-26 2024-09-21T10:00:39.602 INFO:tasks.rados.rados.0.smithi033.stdout:2910: finishing write tid 4 to smithi03385152-26 2024-09-21T10:00:39.602 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1517 (ObjNum 918 snap 267 seq_num 918) dirty exists 2024-09-21T10:00:39.602 INFO:tasks.rados.rados.0.smithi033.stdout:2910: left oid 26 (ObjNum 918 snap 267 seq_num 918) 2024-09-21T10:00:39.604 INFO:tasks.rados.rados.0.smithi033.stdout:2911: finishing copy_from racing read to smithi03385152-45 2024-09-21T10:00:39.605 INFO:tasks.rados.rados.0.smithi033.stdout:2911: finishing copy_from to smithi03385152-45 2024-09-21T10:00:39.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 906 (ObjNum 914 snap 265 seq_num 914) dirty exists 2024-09-21T10:00:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:39 smithi033 ceph-mon[98546]: pgmap v543: 121 pgs: 121 active+clean; 197 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 464 KiB/s wr, 5 op/s 2024-09-21T10:00:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:39 smithi033 ceph-mon[98546]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T10:00:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:39 smithi033 ceph-mon[96568]: pgmap v543: 121 pgs: 121 active+clean; 197 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 464 KiB/s wr, 5 op/s 2024-09-21T10:00:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:39 smithi033 ceph-mon[96568]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T10:00:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:39 smithi105 ceph-mon[86649]: pgmap v543: 121 pgs: 121 active+clean; 197 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 464 KiB/s wr, 5 op/s 2024-09-21T10:00:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:39 smithi105 ceph-mon[86649]: osdmap e610: 8 total, 8 up, 8 in 2024-09-21T10:00:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:00:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:00:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:40.530 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:40 smithi105 podman[115013]: 2024-09-21 10:00:40.143764215 +0000 UTC m=+1.431003545 container remove 6f501c82dfe359e0f9c3e702828bad29f98f989ba91356456d5366e6adf44733 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, io.openshift.tags=base rhel8, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, distribution-scope=public) 2024-09-21T10:00:40.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:40 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:00:40.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:40 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service: Failed with result 'exit-code'. 2024-09-21T10:00:40.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:40 smithi105 systemd[1]: Stopped Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:00:40.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:40 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service: Consumed 7.037s CPU time. 2024-09-21T10:00:40.557 INFO:tasks.rados.rados.0.smithi033.stdout:2910: done (2 left) 2024-09-21T10:00:40.557 INFO:tasks.rados.rados.0.smithi033.stdout:2911: done (1 left) 2024-09-21T10:00:40.557 INFO:tasks.rados.rados.0.smithi033.stdout:2912: done (0 left) 2024-09-21T10:00:40.557 INFO:tasks.rados.rados.0.smithi033.stdout:2913: snap_remove snap 242 2024-09-21T10:00:41.030 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:40 smithi105 systemd[1]: Starting Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:00:41.030 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:40 smithi105 podman[115146]: 2024-09-21 10:00:40.700297636 +0000 UTC m=+0.017256913 image pull 5aad1d7cf004c80a4acb170e1c80fb85781b022f00f08c6bcd22608882b4c748 quay.io/ceph/grafana:10.4.8 2024-09-21T10:00:41.568 INFO:tasks.rados.rados.0.smithi033.stdout:2913: done (0 left) 2024-09-21T10:00:41.568 INFO:tasks.rados.rados.0.smithi033.stdout:2914: snap_remove snap 266 2024-09-21T10:00:41.993 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:41 smithi105 ceph-mon[86649]: pgmap v546: 121 pgs: 121 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 238 KiB/s wr, 10 op/s 2024-09-21T10:00:41.993 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:41 smithi105 ceph-mon[86649]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T10:00:41.993 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:41 smithi105 podman[115146]: 2024-09-21 10:00:41.614036277 +0000 UTC m=+0.930995533 container create 37486b9e0f8e6daa0783f1864670eaec0c98bbd14558ca52bcc45722d31328e1 (image=quay.io/ceph/grafana:10.4.8, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:00:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:00:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:00:41] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-09-21T10:00:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:41 smithi033 ceph-mon[98546]: pgmap v546: 121 pgs: 121 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 238 KiB/s wr, 10 op/s 2024-09-21T10:00:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:41 smithi033 ceph-mon[98546]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T10:00:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:41 smithi033 ceph-mon[96568]: pgmap v546: 121 pgs: 121 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 238 KiB/s wr, 10 op/s 2024-09-21T10:00:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:41 smithi033 ceph-mon[96568]: osdmap e611: 8 total, 8 up, 8 in 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:41 smithi105 podman[115146]: 2024-09-21 10:00:41.99159399 +0000 UTC m=+1.308553252 container init 37486b9e0f8e6daa0783f1864670eaec0c98bbd14558ca52bcc45722d31328e1 (image=quay.io/ceph/grafana:10.4.8, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:41 smithi105 podman[115146]: 2024-09-21 10:00:41.996713122 +0000 UTC m=+1.313672380 container start 37486b9e0f8e6daa0783f1864670eaec0c98bbd14558ca52bcc45722d31328e1 (image=quay.io/ceph/grafana:10.4.8, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 bash[115146]: 37486b9e0f8e6daa0783f1864670eaec0c98bbd14558ca52bcc45722d31328e1 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 systemd[1]: Started Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114529676Z level=info msg="Starting Grafana" version=10.4.8 commit=8c384b71bde142a25ddc01ed5a183ca572b2b7f7 branch=v10.4.x compiled=2024-09-21T10:00:42Z 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.11490386Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114914422Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.11492025Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114925696Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-21T10:00:42.251 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114930866Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-21T10:00:42.252 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114937959Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-21T10:00:42.252 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.1149483Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-21T10:00:42.252 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114955617Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-21T10:00:42.252 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114961291Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-21T10:00:42.252 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114966771Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-21T10:00:42.252 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.11497239Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-21T10:00:42.252 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.114978061Z level=info msg=Target target=[all] 2024-09-21T10:00:42.252 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.11499461Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.115000561Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.11500562Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.115010719Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.115016034Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=settings t=2024-09-21T10:00:42.115021086Z level=info msg="App mode production" 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=sqlstore t=2024-09-21T10:00:42.11536417Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=sqlstore t=2024-09-21T10:00:42.115382919Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.116498581Z level=info msg="Starting DB migrations" 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.127951179Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.133551448Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=5.598722ms 2024-09-21T10:00:42.253 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.13406069Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.134208157Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=146.377µs 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.134607307Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.135357542Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=750.074µs 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.135769614Z level=info msg="Executing migration" id="update login field with orgid to allow for multiple service accounts with same name across orgs" 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.135996772Z level=info msg="Migration successfully executed" id="update login field with orgid to allow for multiple service accounts with same name across orgs" duration=227.165µs 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.136770577Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.141835772Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=5.064388ms 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.142237759Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.147467273Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.229304ms 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.14794509Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.153111338Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=5.174945ms 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.153519494Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-21T10:00:42.254 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.153652351Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=132.872µs 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.154060025Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.154828583Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=768.224µs 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.155240976Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.155951884Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=710.91µs 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.156402816Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.161703044Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.298888ms 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.162125022Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.167190121Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=5.063609ms 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.167580769Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.167634061Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=52.952µs 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.168045899Z level=info msg="Executing migration" id="add library_element folder uid" 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.173118409Z level=info msg="Migration successfully executed" id="add library_element folder uid" duration=5.071353ms 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.173493591Z level=info msg="Executing migration" id="populate library_element folder_uid" 2024-09-21T10:00:42.255 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.173773003Z level=info msg="Migration successfully executed" id="populate library_element folder_uid" duration=279.325µs 2024-09-21T10:00:42.256 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.17433456Z level=info msg="Executing migration" id="add index library_element org_id-folder_uid-name-kind" 2024-09-21T10:00:42.256 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.175221279Z level=info msg="Migration successfully executed" id="add index library_element org_id-folder_uid-name-kind" duration=884.872µs 2024-09-21T10:00:42.257 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.175631485Z level=info msg="Executing migration" id="permission kind migration" 2024-09-21T10:00:42.257 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.180761261Z level=info msg="Migration successfully executed" id="permission kind migration" duration=5.128754ms 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.181141736Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.186174326Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.031647ms 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.186574809Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.191853154Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.278361ms 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.192237504Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.19301882Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=782.315µs 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.193395194Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.194207169Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=811.688µs 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.194583647Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.195351058Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=767.152µs 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.195698138Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-21T10:00:42.258 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.195724644Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=25.101µs 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.196142642Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.19683758Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=694.819µs 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.197181522Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.197889743Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=708.065µs 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.198219309Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.214658492Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=16.436334ms 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.215205839Z level=info msg="Executing migration" id="create correlation v2" 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.215976342Z level=info msg="Migration successfully executed" id="create correlation v2" duration=770.349µs 2024-09-21T10:00:42.259 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.216338377Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.21704703Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=710.814µs 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.217399588Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.218161744Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=761.804µs 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.218528213Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.219254198Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=726.188µs 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.221463813Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.221644195Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=180.343µs 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.224344847Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.22530576Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=961.363µs 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.225782759Z level=info msg="Executing migration" id="add provisioning column" 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.235343036Z level=info msg="Migration successfully executed" id="add provisioning column" duration=9.559191ms 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.235946709Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.236142114Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=194.868µs 2024-09-21T10:00:42.260 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.238677805Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.238751416Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=73.463µs 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.239361372Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.249059334Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=9.695727ms 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.249586206Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.249913919Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=327.844µs 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.250440752Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.250596204Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=155.496µs 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.251132166Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.252565367Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=1.432214ms 2024-09-21T10:00:42.261 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.253039021Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.254124232Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=1.084473ms 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.254715617Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.255863306Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=1.147557ms 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.257653658Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.258272864Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=619.024µs 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.260488048Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.260791144Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=302.874µs 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.261324274Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.262435277Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=1.109456ms 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.262917616Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.264115941Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=1.197844ms 2024-09-21T10:00:42.531 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.265835268Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-21T10:00:42.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.266947575Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=1.112068ms 2024-09-21T10:00:42.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.268827887Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-21T10:00:42.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.270022449Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=1.194076ms 2024-09-21T10:00:42.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.271585896Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-21T10:00:42.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.272655455Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=1.069311ms 2024-09-21T10:00:42.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.274425334Z level=info msg="Executing migration" id="create anon_device table" 2024-09-21T10:00:42.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.275113054Z level=info msg="Migration successfully executed" id="create anon_device table" duration=687.754µs 2024-09-21T10:00:42.532 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.277350294Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.278080426Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=730.002µs 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.280398766Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.281122411Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=723.435µs 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.283252485Z level=info msg="Executing migration" id="create signing_key table" 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.283990611Z level=info msg="Migration successfully executed" id="create signing_key table" duration=737.811µs 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.286097167Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.28680665Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=708.53µs 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.288887915Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.289555467Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=667.509µs 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.291741971Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.292204056Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=462.324µs 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.29261688Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.298138668Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=5.521232ms 2024-09-21T10:00:42.533 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.298517843Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.300323364Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.805873ms 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.300902738Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.30091968Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=17.574µs 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.303649512Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.304439365Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=789.627µs 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.30479978Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.304809051Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=10.033µs 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.30524674Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.306106786Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=859.692µs 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.306495552Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.30733214Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=836.035µs 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.309258027Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.310035911Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=778.304µs 2024-09-21T10:00:42.534 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.312029377Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-21T10:00:42.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.31266544Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=636.038µs 2024-09-21T10:00:42.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.314845328Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-21T10:00:42.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.315269775Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=425.347µs 2024-09-21T10:00:42.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.315605426Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-21T10:00:42.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.315644544Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=39.441µs 2024-09-21T10:00:42.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.316030599Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-21T10:00:42.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.321270796Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.239846ms 2024-09-21T10:00:42.535 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.321603131Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.327027809Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.423908ms 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.327402545Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.327628228Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=225.734µs 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=migrator t=2024-09-21T10:00:42.328047346Z level=info msg="migrations completed" performed=69 skipped=482 duration=200.159856ms 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=sqlstore t=2024-09-21T10:00:42.328702611Z level=info msg="Created default organization" 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=secrets t=2024-09-21T10:00:42.330684737Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=plugin.store t=2024-09-21T10:00:42.345128843Z level=info msg="Loading plugins..." 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=local.finder t=2024-09-21T10:00:42.388949847Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=plugin.store t=2024-09-21T10:00:42.388970795Z level=info msg="Plugins loaded" count=55 duration=43.842748ms 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=query_data t=2024-09-21T10:00:42.391409019Z level=info msg="Query Service initialization" 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=live.push_http t=2024-09-21T10:00:42.396219466Z level=info msg="Live Push Gateway initialization" 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=ngalert.migration t=2024-09-21T10:00:42.39780865Z level=info msg=Starting 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=ngalert.state.manager t=2024-09-21T10:00:42.403692868Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=infra.usagestats.collector t=2024-09-21T10:00:42.40514722Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=provisioning.datasources t=2024-09-21T10:00:42.407669897Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-21T10:00:42.536 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=provisioning.datasources t=2024-09-21T10:00:42.407962947Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=provisioning.alerting t=2024-09-21T10:00:42.449848102Z level=info msg="starting to provision alerting" 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=provisioning.alerting t=2024-09-21T10:00:42.449880044Z level=info msg="finished to provision alerting" 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=ngalert.state.manager t=2024-09-21T10:00:42.450007857Z level=info msg="Warming state cache for startup" 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=ngalert.multiorg.alertmanager t=2024-09-21T10:00:42.450258216Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=grafanaStorageLogger t=2024-09-21T10:00:42.450359054Z level=info msg="Storage starting" 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=http.server t=2024-09-21T10:00:42.452817922Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=http.server t=2024-09-21T10:00:42.453124686Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=provisioning.dashboard t=2024-09-21T10:00:42.480162427Z level=info msg="starting to provision dashboards" 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=ngalert.state.manager t=2024-09-21T10:00:42.4814436Z level=info msg="State cache has been initialized" states=0 duration=31.43464ms 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=ngalert.scheduler t=2024-09-21T10:00:42.481631796Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=ticker t=2024-09-21T10:00:42.481813537Z level=info msg=starting first_tick=2024-09-21T10:00:50Z 2024-09-21T10:00:42.537 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=plugins.update.checker t=2024-09-21T10:00:42.530230968Z level=info msg="Update check succeeded" duration=80.244716ms 2024-09-21T10:00:42.573 INFO:tasks.rados.rados.0.smithi033.stdout:2914: done (0 left) 2024-09-21T10:00:42.573 INFO:tasks.rados.rados.0.smithi033.stdout:2915: rollback oid 35 current snap is 268 2024-09-21T10:00:42.573 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 35 to 263 2024-09-21T10:00:42.573 INFO:tasks.rados.rados.0.smithi033.stdout:2916: snap_remove snap 265 2024-09-21T10:00:42.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-mon[86649]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T10:00:42.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:42.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:42.806 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:42.806 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=grafana.update.checker t=2024-09-21T10:00:42.627238773Z level=info msg="Update check succeeded" duration=176.910822ms 2024-09-21T10:00:42.806 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=provisioning.dashboard t=2024-09-21T10:00:42.662025563Z level=info msg="finished to provision dashboards" 2024-09-21T10:00:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:42 smithi033 ceph-mon[98546]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T10:00:42.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:42 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:42.898 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:42 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:42.898 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:42 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:42 smithi033 ceph-mon[96568]: osdmap e612: 8 total, 8 up, 8 in 2024-09-21T10:00:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:42 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:42 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:42.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:42 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:43.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=grafana-apiserver t=2024-09-21T10:00:42.996346517Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-21T10:00:43.280 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:00:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=grafana-apiserver t=2024-09-21T10:00:42.996725626Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-21T10:00:43.535 INFO:tasks.rados.rados.0.smithi033.stdout:2915: finishing rollback tid 0 to smithi03385152-35 2024-09-21T10:00:43.535 INFO:tasks.rados.rados.0.smithi033.stdout:2915: finishing rollback tid 1 to smithi03385152-35 2024-09-21T10:00:43.535 INFO:tasks.rados.rados.0.smithi033.stdout:2915: finishing rollback tid 2 to smithi03385152-35 2024-09-21T10:00:43.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2174 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:00:43.535 INFO:tasks.rados.rados.0.smithi033.stdout:2915: done (1 left) 2024-09-21T10:00:43.536 INFO:tasks.rados.rados.0.smithi033.stdout:2916: done (0 left) 2024-09-21T10:00:43.536 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 268 2024-09-21T10:00:43.536 INFO:tasks.rados.rados.0.smithi033.stdout:2917: seq_num 919 ranges {0=598016,598016=401408} 2024-09-21T10:00:43.538 INFO:tasks.rados.rados.0.smithi033.stdout:2917: writing smithi03385152-33 from 0 to 598016 tid 1 2024-09-21T10:00:43.541 INFO:tasks.rados.rados.0.smithi033.stdout:2917: writing smithi03385152-33 from 598016 to 999424 tid 2 2024-09-21T10:00:43.541 INFO:tasks.rados.rados.0.smithi033.stdout:2918: setattr oid 11 current snap is 268 2024-09-21T10:00:43.542 INFO:tasks.rados.rados.0.smithi033.stdout:2919: read oid 49 snap -1 2024-09-21T10:00:43.542 INFO:tasks.rados.rados.0.smithi033.stdout:2919: expect deleted 2024-09-21T10:00:43.542 INFO:tasks.rados.rados.0.smithi033.stdout:2920: rollback oid 24 current snap is 268 2024-09-21T10:00:43.542 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 263 2024-09-21T10:00:43.542 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 268 2024-09-21T10:00:43.542 INFO:tasks.rados.rados.0.smithi033.stdout:2921: seq_num 920 ranges {679936=614400,1294336=401408} 2024-09-21T10:00:43.550 INFO:tasks.rados.rados.0.smithi033.stdout:2921: writing smithi03385152-8 from 679936 to 1294336 tid 1 2024-09-21T10:00:43.553 INFO:tasks.rados.rados.0.smithi033.stdout:2921: writing smithi03385152-8 from 1294336 to 1695744 tid 2 2024-09-21T10:00:43.553 INFO:tasks.rados.rados.0.smithi033.stdout:2922: read oid 50 snap -1 2024-09-21T10:00:43.553 INFO:tasks.rados.rados.0.smithi033.stdout:2922: expect (ObjNum 869 snap 254 seq_num 869) 2024-09-21T10:00:43.553 INFO:tasks.rados.rados.0.smithi033.stdout:2923: snap_remove snap 260 2024-09-21T10:00:43.944 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:43 smithi105 ceph-mon[86649]: pgmap v549: 121 pgs: 121 active+clean; 197 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-21T10:00:43.944 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:43 smithi105 ceph-mon[86649]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T10:00:43.944 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:43 smithi105 ceph-mon[86649]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T10:00:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:43 smithi033 ceph-mon[98546]: pgmap v549: 121 pgs: 121 active+clean; 197 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-21T10:00:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:43 smithi033 ceph-mon[98546]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T10:00:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:43 smithi033 ceph-mon[98546]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T10:00:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:43 smithi033 ceph-mon[96568]: pgmap v549: 121 pgs: 121 active+clean; 197 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-21T10:00:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:43 smithi033 ceph-mon[96568]: osdmap e613: 8 total, 8 up, 8 in 2024-09-21T10:00:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:43 smithi033 ceph-mon[96568]: osdmap e614: 8 total, 8 up, 8 in 2024-09-21T10:00:44.534 INFO:tasks.rados.rados.0.smithi033.stdout:2923: done (6 left) 2024-09-21T10:00:44.534 INFO:tasks.rados.rados.0.smithi033.stdout:2924: snap_create 2024-09-21T10:00:44.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1509 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-21T10:00:44.534 INFO:tasks.rados.rados.0.smithi033.stdout:2917: finishing write tid 1 to smithi03385152-33 2024-09-21T10:00:44.534 INFO:tasks.rados.rados.0.smithi033.stdout:2917: finishing write tid 2 to smithi03385152-33 2024-09-21T10:00:44.534 INFO:tasks.rados.rados.0.smithi033.stdout:2917: finishing write tid 3 to smithi03385152-33 2024-09-21T10:00:44.534 INFO:tasks.rados.rados.0.smithi033.stdout:2917: finishing write tid 4 to smithi03385152-33 2024-09-21T10:00:44.534 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1615 (ObjNum 919 snap 268 seq_num 919) dirty exists 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2917: left oid 33 (ObjNum 919 snap 268 seq_num 919) 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2920: finishing rollback tid 0 to smithi03385152-24 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2920: finishing rollback tid 1 to smithi03385152-24 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2920: finishing rollback tid 2 to smithi03385152-24 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 1940 (ObjNum 811 snap 237 seq_num 811) dirty exists 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2921: finishing write tid 1 to smithi03385152-8 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2921: finishing write tid 2 to smithi03385152-8 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2921: finishing write tid 3 to smithi03385152-8 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2921: finishing write tid 4 to smithi03385152-8 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 1494 (ObjNum 920 snap 268 seq_num 920) dirty exists 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2921: left oid 8 (ObjNum 920 snap 268 seq_num 920) 2024-09-21T10:00:44.535 INFO:tasks.rados.rados.0.smithi033.stdout:2922: expect (ObjNum 869 snap 254 seq_num 869) 2024-09-21T10:00:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:2917: done (6 left) 2024-09-21T10:00:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:2918: done (5 left) 2024-09-21T10:00:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:2919: done (4 left) 2024-09-21T10:00:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:2920: done (3 left) 2024-09-21T10:00:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:2921: done (2 left) 2024-09-21T10:00:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:2922: done (1 left) 2024-09-21T10:00:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:2924: done (0 left) 2024-09-21T10:00:45.535 INFO:tasks.rados.rados.0.smithi033.stdout:2925: delete oid 44 current snap is 269 2024-09-21T10:00:45.538 INFO:tasks.rados.rados.0.smithi033.stdout:2925: done (0 left) 2024-09-21T10:00:45.538 INFO:tasks.rados.rados.0.smithi033.stdout:2926: delete oid 20 current snap is 269 2024-09-21T10:00:45.542 INFO:tasks.rados.rados.0.smithi033.stdout:2926: done (0 left) 2024-09-21T10:00:45.542 INFO:tasks.rados.rados.0.smithi033.stdout:2927: rmattr oid 37 current snap is 269 2024-09-21T10:00:45.542 INFO:tasks.rados.rados.0.smithi033.stdout:2927: done (0 left) 2024-09-21T10:00:45.543 INFO:tasks.rados.rados.0.smithi033.stdout:2928: snap_create 2024-09-21T10:00:45.593 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:45 smithi105 ceph-mon[86649]: pgmap v552: 121 pgs: 121 active+clean; 177 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T10:00:45.593 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:45 smithi105 ceph-mon[86649]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T10:00:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:45 smithi033 ceph-mon[98546]: pgmap v552: 121 pgs: 121 active+clean; 177 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T10:00:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:45 smithi033 ceph-mon[98546]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T10:00:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:45 smithi033 ceph-mon[96568]: pgmap v552: 121 pgs: 121 active+clean; 177 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.6 MiB/s wr, 13 op/s 2024-09-21T10:00:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:45 smithi033 ceph-mon[96568]: osdmap e615: 8 total, 8 up, 8 in 2024-09-21T10:00:46.551 INFO:tasks.rados.rados.0.smithi033.stdout:2928: done (0 left) 2024-09-21T10:00:46.551 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 270 2024-09-21T10:00:46.551 INFO:tasks.rados.rados.0.smithi033.stdout:2929: seq_num 923 ranges {40960=663552,704512=401408} 2024-09-21T10:00:46.555 INFO:tasks.rados.rados.0.smithi033.stdout:2929: writing smithi03385152-4 from 40960 to 704512 tid 1 2024-09-21T10:00:46.557 INFO:tasks.rados.rados.0.smithi033.stdout:2929: writing smithi03385152-4 from 704512 to 1105920 tid 2 2024-09-21T10:00:46.557 INFO:tasks.rados.rados.0.smithi033.stdout:2930: read oid 38 snap -1 2024-09-21T10:00:46.557 INFO:tasks.rados.rados.0.smithi033.stdout:2930: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:00:46.557 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 270 2024-09-21T10:00:46.558 INFO:tasks.rados.rados.0.smithi033.stdout:2931: seq_num 924 ranges {0=679936,679936=401408} 2024-09-21T10:00:46.561 INFO:tasks.rados.rados.0.smithi033.stdout:2931: writing smithi03385152-5 from 0 to 679936 tid 1 2024-09-21T10:00:46.564 INFO:tasks.rados.rados.0.smithi033.stdout:2931: writing smithi03385152-5 from 679936 to 1081344 tid 2 2024-09-21T10:00:46.565 INFO:tasks.rados.rados.0.smithi033.stdout:2932: read oid 41 snap 263 2024-09-21T10:00:46.565 INFO:tasks.rados.rados.0.smithi033.stdout:2932: expect deleted 2024-09-21T10:00:46.565 INFO:tasks.rados.rados.0.smithi033.stdout:2933: setattr oid 33 current snap is 270 2024-09-21T10:00:46.565 INFO:tasks.rados.rados.0.smithi033.stdout:2930: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:2929: finishing write tid 1 to smithi03385152-4 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:2929: finishing write tid 2 to smithi03385152-4 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:2929: finishing write tid 3 to smithi03385152-4 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:2929: finishing write tid 4 to smithi03385152-4 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1967 (ObjNum 923 snap 270 seq_num 923) dirty exists 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:2929: left oid 4 (ObjNum 923 snap 270 seq_num 923) 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:2931: finishing write tid 1 to smithi03385152-5 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:2931: finishing write tid 2 to smithi03385152-5 2024-09-21T10:00:46.568 INFO:tasks.rados.rados.0.smithi033.stdout:2931: finishing write tid 3 to smithi03385152-5 2024-09-21T10:00:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:2929: done (4 left) 2024-09-21T10:00:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:2930: done (3 left) 2024-09-21T10:00:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:2932: done (2 left) 2024-09-21T10:00:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:2934: read oid 29 snap -1 2024-09-21T10:00:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:2934: expect deleted 2024-09-21T10:00:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:2935: copy_from oid 2 from oid 49 current snap is 270 2024-09-21T10:00:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:2936: snap_create 2024-09-21T10:00:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:2931: finishing write tid 4 to smithi03385152-5 2024-09-21T10:00:46.570 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 1054 (ObjNum 924 snap 270 seq_num 924) dirty exists 2024-09-21T10:00:46.570 INFO:tasks.rados.rados.0.smithi033.stdout:2931: left oid 5 (ObjNum 924 snap 270 seq_num 924) 2024-09-21T10:00:46.570 INFO:tasks.rados.rados.0.smithi033.stdout:2935: finishing copy_from to smithi03385152-2 2024-09-21T10:00:46.570 INFO:tasks.rados.rados.0.smithi033.stdout:2935: got expected ENOENT (src dne) 2024-09-21T10:00:46.570 INFO:tasks.rados.rados.0.smithi033.stdout:2935: finishing copy_from racing read to smithi03385152-2 2024-09-21T10:00:46.570 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1616 (ObjNum 919 snap 268 seq_num 919) dirty exists 2024-09-21T10:00:46.859 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:46 smithi105 ceph-mon[86649]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T10:00:46.859 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:46.859 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.859 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.859 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.859 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[98546]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T10:00:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[96568]: osdmap e616: 8 total, 8 up, 8 in 2024-09-21T10:00:46.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:00:46.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:46.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:47.563 INFO:tasks.rados.rados.0.smithi033.stdout:2931: done (4 left) 2024-09-21T10:00:47.563 INFO:tasks.rados.rados.0.smithi033.stdout:2933: done (3 left) 2024-09-21T10:00:47.563 INFO:tasks.rados.rados.0.smithi033.stdout:2934: done (2 left) 2024-09-21T10:00:47.563 INFO:tasks.rados.rados.0.smithi033.stdout:2935: done (1 left) 2024-09-21T10:00:47.563 INFO:tasks.rados.rados.0.smithi033.stdout:2936: done (0 left) 2024-09-21T10:00:47.563 INFO:tasks.rados.rados.0.smithi033.stdout:2937: delete oid 22 current snap is 271 2024-09-21T10:00:47.568 INFO:tasks.rados.rados.0.smithi033.stdout:2937: done (0 left) 2024-09-21T10:00:47.568 INFO:tasks.rados.rados.0.smithi033.stdout:2938: rollback oid 19 current snap is 271 2024-09-21T10:00:47.568 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 19 to 268 2024-09-21T10:00:47.568 INFO:tasks.rados.rados.0.smithi033.stdout:2939: rmattr oid 23 current snap is 271 2024-09-21T10:00:47.568 INFO:tasks.rados.rados.0.smithi033.stdout:2939: done (1 left) 2024-09-21T10:00:47.568 INFO:tasks.rados.rados.0.smithi033.stdout:2940: read oid 50 snap -1 2024-09-21T10:00:47.568 INFO:tasks.rados.rados.0.smithi033.stdout:2940: expect (ObjNum 869 snap 254 seq_num 869) 2024-09-21T10:00:47.568 INFO:tasks.rados.rados.0.smithi033.stdout:2941: delete oid 10 current snap is 271 2024-09-21T10:00:47.572 INFO:tasks.rados.rados.0.smithi033.stdout:2940: expect (ObjNum 869 snap 254 seq_num 869) 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:2938: finishing rollback tid 0 to smithi03385152-19 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:2938: finishing rollback tid 1 to smithi03385152-19 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:2938: finishing rollback tid 2 to smithi03385152-19 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 1943 (ObjNum 877 snap 255 seq_num 877) dirty exists 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:2938: done (2 left) 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:2940: done (1 left) 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:2941: done (0 left) 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:2942: rmattr oid 7 current snap is 271 2024-09-21T10:00:47.577 INFO:tasks.rados.rados.0.smithi033.stdout:2943: snap_remove snap 261 2024-09-21T10:00:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:47 smithi105 ceph-mon[86649]: pgmap v555: 121 pgs: 121 active+clean; 116 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 484 KiB/s wr, 9 op/s 2024-09-21T10:00:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:47 smithi105 ceph-mon[86649]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T10:00:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:47 smithi033 ceph-mon[98546]: pgmap v555: 121 pgs: 121 active+clean; 116 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 484 KiB/s wr, 9 op/s 2024-09-21T10:00:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:47 smithi033 ceph-mon[98546]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T10:00:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:47 smithi033 ceph-mon[96568]: pgmap v555: 121 pgs: 121 active+clean; 116 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 484 KiB/s wr, 9 op/s 2024-09-21T10:00:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:47 smithi033 ceph-mon[96568]: osdmap e617: 8 total, 8 up, 8 in 2024-09-21T10:00:48.568 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2182 (ObjNum 724 snap 205 seq_num 724) dirty exists 2024-09-21T10:00:48.568 INFO:tasks.rados.rados.0.smithi033.stdout:2942: done (1 left) 2024-09-21T10:00:48.568 INFO:tasks.rados.rados.0.smithi033.stdout:2943: done (0 left) 2024-09-21T10:00:48.568 INFO:tasks.rados.rados.0.smithi033.stdout:2944: rollback oid 47 current snap is 271 2024-09-21T10:00:48.568 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 268 2024-09-21T10:00:48.568 INFO:tasks.rados.rados.0.smithi033.stdout:2945: copy_from oid 35 from oid 44 current snap is 271 2024-09-21T10:00:48.568 INFO:tasks.rados.rados.0.smithi033.stdout:2946: read oid 32 snap -1 2024-09-21T10:00:48.568 INFO:tasks.rados.rados.0.smithi033.stdout:2946: expect (ObjNum 900 snap 260 seq_num 900) 2024-09-21T10:00:48.569 INFO:tasks.rados.rados.0.smithi033.stdout:2947: snap_create 2024-09-21T10:00:48.575 INFO:tasks.rados.rados.0.smithi033.stdout:2945: finishing copy_from to smithi03385152-35 2024-09-21T10:00:48.575 INFO:tasks.rados.rados.0.smithi033.stdout:2945: got expected ENOENT (src dne) 2024-09-21T10:00:48.575 INFO:tasks.rados.rados.0.smithi033.stdout:2945: finishing copy_from racing read to smithi03385152-35 2024-09-21T10:00:48.576 INFO:tasks.rados.rados.0.smithi033.stdout:2944: finishing rollback tid 1 to smithi03385152-47 2024-09-21T10:00:48.576 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1494 (ObjNum 33 snap 0 seq_num 91608144) dirty dne 2024-09-21T10:00:48.601 INFO:tasks.rados.rados.0.smithi033.stdout:2946: expect (ObjNum 900 snap 260 seq_num 900) 2024-09-21T10:00:48.856 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:48 smithi105 ceph-mon[86649]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T10:00:48.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:48 smithi033 ceph-mon[98546]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T10:00:48.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:48 smithi033 ceph-mon[96568]: osdmap e618: 8 total, 8 up, 8 in 2024-09-21T10:00:49.569 INFO:tasks.rados.rados.0.smithi033.stdout:2944: done (3 left) 2024-09-21T10:00:49.569 INFO:tasks.rados.rados.0.smithi033.stdout:2945: done (2 left) 2024-09-21T10:00:49.569 INFO:tasks.rados.rados.0.smithi033.stdout:2946: done (1 left) 2024-09-21T10:00:49.569 INFO:tasks.rados.rados.0.smithi033.stdout:2947: done (0 left) 2024-09-21T10:00:49.569 INFO:tasks.rados.rados.0.smithi033.stdout:2948: rollback oid 43 current snap is 272 2024-09-21T10:00:49.569 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 43 to 269 2024-09-21T10:00:49.569 INFO:tasks.rados.rados.0.smithi033.stdout:2949: copy_from oid 32 from oid 46 current snap is 272 2024-09-21T10:00:49.569 INFO:tasks.rados.rados.0.smithi033.stdout:2950: copy_from oid 33 from oid 40 current snap is 272 2024-09-21T10:00:49.570 INFO:tasks.rados.rados.0.smithi033.stdout:2951: snap_remove snap 268 2024-09-21T10:00:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:49 smithi033 ceph-mon[98546]: pgmap v558: 121 pgs: 121 active+clean; 113 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 248 KiB/s wr, 7 op/s 2024-09-21T10:00:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:49 smithi033 ceph-mon[98546]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T10:00:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:49 smithi033 ceph-mon[96568]: pgmap v558: 121 pgs: 121 active+clean; 113 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 248 KiB/s wr, 7 op/s 2024-09-21T10:00:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:49 smithi033 ceph-mon[96568]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T10:00:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:49 smithi105 ceph-mon[86649]: pgmap v558: 121 pgs: 121 active+clean; 113 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 248 KiB/s wr, 7 op/s 2024-09-21T10:00:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:49 smithi105 ceph-mon[86649]: osdmap e619: 8 total, 8 up, 8 in 2024-09-21T10:00:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:00:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:00:50.584 INFO:tasks.rados.rados.0.smithi033.stdout:2951: done (3 left) 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2952: delete oid 29 current snap is 272 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2948: finishing rollback tid 0 to smithi03385152-43 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2948: finishing rollback tid 1 to smithi03385152-43 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2948: finishing rollback tid 2 to smithi03385152-43 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1512 (ObjNum 909 snap 263 seq_num 909) dirty exists 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2950: finishing copy_from to smithi03385152-33 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2950: got expected ENOENT (src dne) 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2950: finishing copy_from racing read to smithi03385152-33 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2949: finishing copy_from racing read to smithi03385152-32 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:2949: finishing copy_from to smithi03385152-32 2024-09-21T10:00:50.585 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1582 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:00:50.595 INFO:tasks.rados.rados.0.smithi033.stdout:2948: done (3 left) 2024-09-21T10:00:50.595 INFO:tasks.rados.rados.0.smithi033.stdout:2949: done (2 left) 2024-09-21T10:00:50.596 INFO:tasks.rados.rados.0.smithi033.stdout:2950: done (1 left) 2024-09-21T10:00:50.596 INFO:tasks.rados.rados.0.smithi033.stdout:2952: done (0 left) 2024-09-21T10:00:50.596 INFO:tasks.rados.rados.0.smithi033.stdout:2953: delete oid 21 current snap is 272 2024-09-21T10:00:50.598 INFO:tasks.rados.rados.0.smithi033.stdout:2953: done (0 left) 2024-09-21T10:00:50.598 INFO:tasks.rados.rados.0.smithi033.stdout:2954: copy_from oid 13 from oid 17 current snap is 272 2024-09-21T10:00:50.598 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 272 2024-09-21T10:00:50.598 INFO:tasks.rados.rados.0.smithi033.stdout:2955: seq_num 929 ranges {2613248=761856,3375104=401408} 2024-09-21T10:00:50.616 INFO:tasks.rados.rados.0.smithi033.stdout:2955: writing smithi03385152-26 from 2613248 to 3375104 tid 1 2024-09-21T10:00:50.619 INFO:tasks.rados.rados.0.smithi033.stdout:2955: writing smithi03385152-26 from 3375104 to 3776512 tid 2 2024-09-21T10:00:50.619 INFO:tasks.rados.rados.0.smithi033.stdout:2956: snap_create 2024-09-21T10:00:50.619 INFO:tasks.rados.rados.0.smithi033.stdout:2954: finishing copy_from to smithi03385152-13 2024-09-21T10:00:50.619 INFO:tasks.rados.rados.0.smithi033.stdout:2954: got expected ENOENT (src dne) 2024-09-21T10:00:50.619 INFO:tasks.rados.rados.0.smithi033.stdout:2954: finishing copy_from racing read to smithi03385152-13 2024-09-21T10:00:50.621 INFO:tasks.rados.rados.0.smithi033.stdout:2955: finishing write tid 1 to smithi03385152-26 2024-09-21T10:00:50.622 INFO:tasks.rados.rados.0.smithi033.stdout:2955: finishing write tid 2 to smithi03385152-26 2024-09-21T10:00:50.622 INFO:tasks.rados.rados.0.smithi033.stdout:2955: finishing write tid 3 to smithi03385152-26 2024-09-21T10:00:50.623 INFO:tasks.rados.rados.0.smithi033.stdout:2955: finishing write tid 4 to smithi03385152-26 2024-09-21T10:00:50.623 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1520 (ObjNum 929 snap 272 seq_num 929) dirty exists 2024-09-21T10:00:50.623 INFO:tasks.rados.rados.0.smithi033.stdout:2955: left oid 26 (ObjNum 929 snap 272 seq_num 929) 2024-09-21T10:00:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T10:00:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: Upgrade: Finalizing container_image settings 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:50.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:00:50.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:51.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.000 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: Upgrade: Finalizing container_image settings 2024-09-21T10:00:51.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:00:51.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:00:51.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:00:51.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:00:51.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:00:51.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:00:51.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T10:00:51.003 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:00:51.004 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T10:00:51.005 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.006 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: Upgrade: Complete! 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.007 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: Upgrade: Complete! 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:51.008 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: osdmap e620: 8 total, 8 up, 8 in 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "dashboard get-grafana-api-url"} : dispatch 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:00:51.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: Upgrade: Finalizing container_image settings 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-21T10:00:51.033 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-21T10:00:51.034 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: Upgrade: Complete! 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:00:51.035 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:00:51.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:00:51.036 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.593 INFO:tasks.rados.rados.0.smithi033.stdout:2954: done (2 left) 2024-09-21T10:00:51.594 INFO:tasks.rados.rados.0.smithi033.stdout:2955: done (1 left) 2024-09-21T10:00:51.594 INFO:tasks.rados.rados.0.smithi033.stdout:2956: done (0 left) 2024-09-21T10:00:51.594 INFO:tasks.rados.rados.0.smithi033.stdout:2957: copy_from oid 24 from oid 35 current snap is 273 2024-09-21T10:00:51.594 INFO:tasks.rados.rados.0.smithi033.stdout:2958: read oid 32 snap 263 2024-09-21T10:00:51.594 INFO:tasks.rados.rados.0.smithi033.stdout:2958: expect (ObjNum 900 snap 260 seq_num 900) 2024-09-21T10:00:51.594 INFO:tasks.rados.rados.0.smithi033.stdout:2959: delete oid 31 current snap is 273 2024-09-21T10:00:51.602 INFO:tasks.rados.rados.0.smithi033.stdout:2959: done (2 left) 2024-09-21T10:00:51.602 INFO:tasks.rados.rados.0.smithi033.stdout:2960: rollback oid 11 current snap is 273 2024-09-21T10:00:51.602 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 11 to 271 2024-09-21T10:00:51.602 INFO:tasks.rados.rados.0.smithi033.stdout:2961: delete oid 44 current snap is 273 2024-09-21T10:00:51.606 INFO:tasks.rados.rados.0.smithi033.stdout:2960: finishing rollback tid 0 to smithi03385152-11 2024-09-21T10:00:51.606 INFO:tasks.rados.rados.0.smithi033.stdout:2960: finishing rollback tid 1 to smithi03385152-11 2024-09-21T10:00:51.606 INFO:tasks.rados.rados.0.smithi033.stdout:2961: done (3 left) 2024-09-21T10:00:51.606 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 273 2024-09-21T10:00:51.607 INFO:tasks.rados.rados.0.smithi033.stdout:2962: seq_num 932 ranges {516096=417792,933888=663552,1597440=139264} 2024-09-21T10:00:51.613 INFO:tasks.rados.rados.0.smithi033.stdout:2962: writing smithi03385152-3 from 516096 to 933888 tid 1 2024-09-21T10:00:51.616 INFO:tasks.rados.rados.0.smithi033.stdout:2962: writing smithi03385152-3 from 933888 to 1597440 tid 2 2024-09-21T10:00:51.617 INFO:tasks.rados.rados.0.smithi033.stdout:2962: writing smithi03385152-3 from 1597440 to 1736704 tid 3 2024-09-21T10:00:51.617 INFO:tasks.rados.rados.0.smithi033.stdout:2963: snap_remove snap 267 2024-09-21T10:00:51.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:00:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:00:51] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-09-21T10:00:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:51 smithi033 ceph-mon[98546]: pgmap v561: 121 pgs: 121 active+clean; 110 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 772 KiB/s wr, 9 op/s 2024-09-21T10:00:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:51 smithi033 ceph-mon[98546]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T10:00:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:51 smithi033 ceph-mon[96568]: pgmap v561: 121 pgs: 121 active+clean; 110 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 772 KiB/s wr, 9 op/s 2024-09-21T10:00:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:51 smithi033 ceph-mon[96568]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T10:00:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:51 smithi105 ceph-mon[86649]: pgmap v561: 121 pgs: 121 active+clean; 110 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 772 KiB/s wr, 9 op/s 2024-09-21T10:00:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:51 smithi105 ceph-mon[86649]: osdmap e621: 8 total, 8 up, 8 in 2024-09-21T10:00:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:00:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:2963: done (4 left) 2024-09-21T10:00:52.605 INFO:tasks.rados.rados.0.smithi033.stdout:2960: finishing rollback tid 2 to smithi03385152-11 2024-09-21T10:00:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 1515 (ObjNum 129 snap 0 seq_num 91360152) dirty exists 2024-09-21T10:00:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:2957: finishing copy_from racing read to smithi03385152-24 2024-09-21T10:00:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:2962: finishing write tid 1 to smithi03385152-3 2024-09-21T10:00:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:2962: finishing write tid 2 to smithi03385152-3 2024-09-21T10:00:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:2962: finishing write tid 3 to smithi03385152-3 2024-09-21T10:00:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:2962: finishing write tid 4 to smithi03385152-3 2024-09-21T10:00:52.606 INFO:tasks.rados.rados.0.smithi033.stdout:2958: expect (ObjNum 900 snap 260 seq_num 900) 2024-09-21T10:00:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:52 smithi033 ceph-mon[98546]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T10:00:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:52 smithi033 ceph-mon[96568]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T10:00:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:52 smithi105 ceph-mon[86649]: osdmap e622: 8 total, 8 up, 8 in 2024-09-21T10:00:53.195 INFO:tasks.rados.rados.0.smithi033.stdout:2962: finishing write tid 5 to smithi03385152-3 2024-09-21T10:00:53.195 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1783 (ObjNum 932 snap 273 seq_num 932) dirty exists 2024-09-21T10:00:53.195 INFO:tasks.rados.rados.0.smithi033.stdout:2962: left oid 3 (ObjNum 932 snap 273 seq_num 932) 2024-09-21T10:00:53.195 INFO:tasks.rados.rados.0.smithi033.stdout:2957: finishing copy_from to smithi03385152-24 2024-09-21T10:00:53.195 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 2174 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:00:53.196 INFO:tasks.rados.rados.0.smithi033.stdout:2964: read oid 30 snap -1 2024-09-21T10:00:53.196 INFO:tasks.rados.rados.0.smithi033.stdout:2964: expect (ObjNum 811 snap 237 seq_num 811) 2024-09-21T10:00:53.196 INFO:tasks.rados.rados.0.smithi033.stdout:2957: done (4 left) 2024-09-21T10:00:53.196 INFO:tasks.rados.rados.0.smithi033.stdout:2958: done (3 left) 2024-09-21T10:00:53.196 INFO:tasks.rados.rados.0.smithi033.stdout:2960: done (2 left) 2024-09-21T10:00:53.196 INFO:tasks.rados.rados.0.smithi033.stdout:2962: done (1 left) 2024-09-21T10:00:53.196 INFO:tasks.rados.rados.0.smithi033.stdout:2965: snap_remove snap 264 2024-09-21T10:00:53.537 INFO:tasks.rados.rados.0.smithi033.stdout:2965: done (1 left) 2024-09-21T10:00:53.537 INFO:tasks.rados.rados.0.smithi033.stdout:2966: read oid 38 snap -1 2024-09-21T10:00:53.537 INFO:tasks.rados.rados.0.smithi033.stdout:2966: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:00:53.537 INFO:tasks.rados.rados.0.smithi033.stdout:2967: delete oid 4 current snap is 273 2024-09-21T10:00:53.537 INFO:tasks.rados.rados.0.smithi033.stdout:2964: expect (ObjNum 811 snap 237 seq_num 811) 2024-09-21T10:00:53.585 INFO:tasks.rados.rados.0.smithi033.stdout:2966: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:00:53.588 INFO:tasks.rados.rados.0.smithi033.stdout:2964: done (2 left) 2024-09-21T10:00:53.588 INFO:tasks.rados.rados.0.smithi033.stdout:2966: done (1 left) 2024-09-21T10:00:53.588 INFO:tasks.rados.rados.0.smithi033.stdout:2967: done (0 left) 2024-09-21T10:00:53.588 INFO:tasks.rados.rados.0.smithi033.stdout:2968: rollback oid 46 current snap is 273 2024-09-21T10:00:53.588 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 46 to 269 2024-09-21T10:00:53.588 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 273 2024-09-21T10:00:53.588 INFO:tasks.rados.rados.0.smithi033.stdout:2969: seq_num 934 ranges {0=450560,450560=434176,884736=368640} 2024-09-21T10:00:53.590 INFO:tasks.rados.rados.0.smithi033.stdout:2969: writing smithi03385152-47 from 0 to 450560 tid 1 2024-09-21T10:00:53.593 INFO:tasks.rados.rados.0.smithi033.stdout:2969: writing smithi03385152-47 from 450560 to 884736 tid 2 2024-09-21T10:00:53.596 INFO:tasks.rados.rados.0.smithi033.stdout:2969: writing smithi03385152-47 from 884736 to 1253376 tid 3 2024-09-21T10:00:53.596 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 273 2024-09-21T10:00:53.596 INFO:tasks.rados.rados.0.smithi033.stdout:2968: finishing rollback tid 0 to smithi03385152-46 2024-09-21T10:00:53.596 INFO:tasks.rados.rados.0.smithi033.stdout:2968: finishing rollback tid 1 to smithi03385152-46 2024-09-21T10:00:53.596 INFO:tasks.rados.rados.0.smithi033.stdout:2968: finishing rollback tid 2 to smithi03385152-46 2024-09-21T10:00:53.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1786 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:00:53.597 INFO:tasks.rados.rados.0.smithi033.stdout:2969: finishing write tid 1 to smithi03385152-47 2024-09-21T10:00:53.597 INFO:tasks.rados.rados.0.smithi033.stdout:2969: finishing write tid 2 to smithi03385152-47 2024-09-21T10:00:53.597 INFO:tasks.rados.rados.0.smithi033.stdout:2970: seq_num 935 ranges {8839168=466944,9306112=720896,10027008=81920} 2024-09-21T10:00:53.646 INFO:tasks.rados.rados.0.smithi033.stdout:2970: writing smithi03385152-14 from 8839168 to 9306112 tid 1 2024-09-21T10:00:53.650 INFO:tasks.rados.rados.0.smithi033.stdout:2970: writing smithi03385152-14 from 9306112 to 10027008 tid 2 2024-09-21T10:00:53.651 INFO:tasks.rados.rados.0.smithi033.stdout:2970: writing smithi03385152-14 from 10027008 to 10108928 tid 3 2024-09-21T10:00:53.651 INFO:tasks.rados.rados.0.smithi033.stdout:2968: done (2 left) 2024-09-21T10:00:53.651 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 273 2024-09-21T10:00:53.652 INFO:tasks.rados.rados.0.smithi033.stdout:2969: finishing write tid 3 to smithi03385152-47 2024-09-21T10:00:53.652 INFO:tasks.rados.rados.0.smithi033.stdout:2969: finishing write tid 4 to smithi03385152-47 2024-09-21T10:00:53.652 INFO:tasks.rados.rados.0.smithi033.stdout:2969: finishing write tid 5 to smithi03385152-47 2024-09-21T10:00:53.652 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1498 (ObjNum 934 snap 273 seq_num 934) dirty exists 2024-09-21T10:00:53.652 INFO:tasks.rados.rados.0.smithi033.stdout:2969: left oid 47 (ObjNum 934 snap 273 seq_num 934) 2024-09-21T10:00:53.652 INFO:tasks.rados.rados.0.smithi033.stdout:2970: finishing write tid 1 to smithi03385152-14 2024-09-21T10:00:53.652 INFO:tasks.rados.rados.0.smithi033.stdout:2971: seq_num 936 ranges {0=483328,483328=655360,1138688=147456} 2024-09-21T10:00:53.653 INFO:tasks.rados.rados.0.smithi033.stdout:2971: writing smithi03385152-18 from 0 to 483328 tid 1 2024-09-21T10:00:53.656 INFO:tasks.rados.rados.0.smithi033.stdout:2971: writing smithi03385152-18 from 483328 to 1138688 tid 2 2024-09-21T10:00:53.658 INFO:tasks.rados.rados.0.smithi033.stdout:2971: writing smithi03385152-18 from 1138688 to 1286144 tid 3 2024-09-21T10:00:53.658 INFO:tasks.rados.rados.0.smithi033.stdout:2969: done (2 left) 2024-09-21T10:00:53.658 INFO:tasks.rados.rados.0.smithi033.stdout:2972: snap_create 2024-09-21T10:00:53.658 INFO:tasks.rados.rados.0.smithi033.stdout:2970: finishing write tid 2 to smithi03385152-14 2024-09-21T10:00:53.658 INFO:tasks.rados.rados.0.smithi033.stdout:2970: finishing write tid 3 to smithi03385152-14 2024-09-21T10:00:53.659 INFO:tasks.rados.rados.0.smithi033.stdout:2971: finishing write tid 1 to smithi03385152-18 2024-09-21T10:00:53.659 INFO:tasks.rados.rados.0.smithi033.stdout:2970: finishing write tid 4 to smithi03385152-14 2024-09-21T10:00:53.659 INFO:tasks.rados.rados.0.smithi033.stdout:2970: finishing write tid 5 to smithi03385152-14 2024-09-21T10:00:53.659 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1790 (ObjNum 935 snap 273 seq_num 935) dirty exists 2024-09-21T10:00:53.659 INFO:tasks.rados.rados.0.smithi033.stdout:2970: left oid 14 (ObjNum 935 snap 273 seq_num 935) 2024-09-21T10:00:53.660 INFO:tasks.rados.rados.0.smithi033.stdout:2971: finishing write tid 2 to smithi03385152-18 2024-09-21T10:00:53.660 INFO:tasks.rados.rados.0.smithi033.stdout:2971: finishing write tid 3 to smithi03385152-18 2024-09-21T10:00:53.660 INFO:tasks.rados.rados.0.smithi033.stdout:2971: finishing write tid 4 to smithi03385152-18 2024-09-21T10:00:53.665 INFO:tasks.rados.rados.0.smithi033.stdout:2971: finishing write tid 5 to smithi03385152-18 2024-09-21T10:00:53.665 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1020 (ObjNum 936 snap 273 seq_num 936) dirty exists 2024-09-21T10:00:53.665 INFO:tasks.rados.rados.0.smithi033.stdout:2971: left oid 18 (ObjNum 936 snap 273 seq_num 936) 2024-09-21T10:00:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:53 smithi033 ceph-mon[98546]: pgmap v564: 121 pgs: 121 active+clean; 104 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 524 KiB/s wr, 10 op/s 2024-09-21T10:00:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:53 smithi033 ceph-mon[98546]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T10:00:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:53 smithi033 ceph-mon[98546]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T10:00:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:53 smithi033 ceph-mon[96568]: pgmap v564: 121 pgs: 121 active+clean; 104 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 524 KiB/s wr, 10 op/s 2024-09-21T10:00:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:53 smithi033 ceph-mon[96568]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T10:00:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:53 smithi033 ceph-mon[96568]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T10:00:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:53 smithi105 ceph-mon[86649]: pgmap v564: 121 pgs: 121 active+clean; 104 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 524 KiB/s wr, 10 op/s 2024-09-21T10:00:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:53 smithi105 ceph-mon[86649]: osdmap e623: 8 total, 8 up, 8 in 2024-09-21T10:00:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:53 smithi105 ceph-mon[86649]: osdmap e624: 8 total, 8 up, 8 in 2024-09-21T10:00:54.610 INFO:tasks.rados.rados.0.smithi033.stdout:2970: done (2 left) 2024-09-21T10:00:54.610 INFO:tasks.rados.rados.0.smithi033.stdout:2971: done (1 left) 2024-09-21T10:00:54.610 INFO:tasks.rados.rados.0.smithi033.stdout:2972: done (0 left) 2024-09-21T10:00:54.610 INFO:tasks.rados.rados.0.smithi033.stdout:2973: copy_from oid 48 from oid 43 current snap is 274 2024-09-21T10:00:54.610 INFO:tasks.rados.rados.0.smithi033.stdout:2974: setattr oid 7 current snap is 274 2024-09-21T10:00:54.611 INFO:tasks.rados.rados.0.smithi033.stdout:2975: read oid 36 snap 272 2024-09-21T10:00:54.611 INFO:tasks.rados.rados.0.smithi033.stdout:2975: expect (ObjNum 91039504 snap 21955 seq_num 91360192) 2024-09-21T10:00:54.611 INFO:tasks.rados.rados.0.smithi033.stdout:2976: delete oid 20 current snap is 274 2024-09-21T10:00:54.613 INFO:tasks.rados.rados.0.smithi033.stdout:2976: done (3 left) 2024-09-21T10:00:54.613 INFO:tasks.rados.rados.0.smithi033.stdout:2977: snap_remove snap 263 2024-09-21T10:00:55.612 INFO:tasks.rados.rados.0.smithi033.stdout:2975: expect (ObjNum 91039504 snap 21955 seq_num 91360192) 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2207 (ObjNum 724 snap 205 seq_num 724) dirty exists 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:2973: finishing copy_from racing read to smithi03385152-48 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:2973: finishing copy_from to smithi03385152-48 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 1512 (ObjNum 909 snap 263 seq_num 909) dirty exists 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:2973: done (3 left) 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:2974: done (2 left) 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:2975: done (1 left) 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:2977: done (0 left) 2024-09-21T10:00:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:2978: delete oid 35 current snap is 274 2024-09-21T10:00:55.622 INFO:tasks.rados.rados.0.smithi033.stdout:2978: done (0 left) 2024-09-21T10:00:55.622 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 274 2024-09-21T10:00:55.622 INFO:tasks.rados.rados.0.smithi033.stdout:2979: seq_num 939 ranges {0=532480,532480=704512,1236992=98304} 2024-09-21T10:00:55.625 INFO:tasks.rados.rados.0.smithi033.stdout:2979: writing smithi03385152-44 from 0 to 532480 tid 1 2024-09-21T10:00:55.629 INFO:tasks.rados.rados.0.smithi033.stdout:2979: writing smithi03385152-44 from 532480 to 1236992 tid 2 2024-09-21T10:00:55.630 INFO:tasks.rados.rados.0.smithi033.stdout:2979: writing smithi03385152-44 from 1236992 to 1335296 tid 3 2024-09-21T10:00:55.630 INFO:tasks.rados.rados.0.smithi033.stdout:2980: snap_create 2024-09-21T10:00:55.630 INFO:tasks.rados.rados.0.smithi033.stdout:2979: finishing write tid 1 to smithi03385152-44 2024-09-21T10:00:55.633 INFO:tasks.rados.rados.0.smithi033.stdout:2979: finishing write tid 2 to smithi03385152-44 2024-09-21T10:00:55.634 INFO:tasks.rados.rados.0.smithi033.stdout:2979: finishing write tid 3 to smithi03385152-44 2024-09-21T10:00:55.635 INFO:tasks.rados.rados.0.smithi033.stdout:2979: finishing write tid 4 to smithi03385152-44 2024-09-21T10:00:55.636 INFO:tasks.rados.rados.0.smithi033.stdout:2979: finishing write tid 5 to smithi03385152-44 2024-09-21T10:00:55.636 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1972 (ObjNum 939 snap 274 seq_num 939) dirty exists 2024-09-21T10:00:55.636 INFO:tasks.rados.rados.0.smithi033.stdout:2979: left oid 44 (ObjNum 939 snap 274 seq_num 939) 2024-09-21T10:00:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:55 smithi033 ceph-mon[98546]: pgmap v567: 121 pgs: 121 active+clean; 99 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 682 KiB/s wr, 7 op/s 2024-09-21T10:00:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:55 smithi033 ceph-mon[98546]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T10:00:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:55 smithi033 ceph-mon[96568]: pgmap v567: 121 pgs: 121 active+clean; 99 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 682 KiB/s wr, 7 op/s 2024-09-21T10:00:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:55 smithi033 ceph-mon[96568]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T10:00:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:55 smithi105 ceph-mon[86649]: pgmap v567: 121 pgs: 121 active+clean; 99 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 682 KiB/s wr, 7 op/s 2024-09-21T10:00:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:55 smithi105 ceph-mon[86649]: osdmap e625: 8 total, 8 up, 8 in 2024-09-21T10:00:56.627 INFO:tasks.rados.rados.0.smithi033.stdout:2979: done (1 left) 2024-09-21T10:00:56.628 INFO:tasks.rados.rados.0.smithi033.stdout:2980: done (0 left) 2024-09-21T10:00:56.628 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 275 2024-09-21T10:00:56.628 INFO:tasks.rados.rados.0.smithi033.stdout:2981: seq_num 940 ranges {0=548864,548864=589824,1138688=212992} 2024-09-21T10:00:56.630 INFO:tasks.rados.rados.0.smithi033.stdout:2981: writing smithi03385152-13 from 0 to 548864 tid 1 2024-09-21T10:00:56.634 INFO:tasks.rados.rados.0.smithi033.stdout:2981: writing smithi03385152-13 from 548864 to 1138688 tid 2 2024-09-21T10:00:56.635 INFO:tasks.rados.rados.0.smithi033.stdout:2981: writing smithi03385152-13 from 1138688 to 1351680 tid 3 2024-09-21T10:00:56.635 INFO:tasks.rados.rados.0.smithi033.stdout:2982: copy_from oid 41 from oid 46 current snap is 275 2024-09-21T10:00:56.635 INFO:tasks.rados.rados.0.smithi033.stdout:2983: snap_remove snap 269 2024-09-21T10:00:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:56 smithi033 ceph-mon[98546]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T10:00:56.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:56 smithi033 ceph-mon[96568]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T10:00:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:56 smithi105 ceph-mon[86649]: osdmap e626: 8 total, 8 up, 8 in 2024-09-21T10:00:57.636 INFO:tasks.rados.rados.0.smithi033.stdout:2983: done (2 left) 2024-09-21T10:00:57.636 INFO:tasks.rados.rados.0.smithi033.stdout:2984: read oid 23 snap -1 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2984: expect deleted 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2985: rollback oid 28 current snap is 275 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 273 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2981: finishing write tid 1 to smithi03385152-13 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2981: finishing write tid 2 to smithi03385152-13 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2981: finishing write tid 3 to smithi03385152-13 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2981: finishing write tid 4 to smithi03385152-13 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2981: finishing write tid 5 to smithi03385152-13 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2221 (ObjNum 940 snap 275 seq_num 940) dirty exists 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2981: left oid 13 (ObjNum 940 snap 275 seq_num 940) 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2982: finishing copy_from racing read to smithi03385152-41 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2982: finishing copy_from to smithi03385152-41 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 1786 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2981: done (3 left) 2024-09-21T10:00:57.637 INFO:tasks.rados.rados.0.smithi033.stdout:2982: done (2 left) 2024-09-21T10:00:57.638 INFO:tasks.rados.rados.0.smithi033.stdout:2986: setattr oid 21 current snap is 275 2024-09-21T10:00:57.638 INFO:tasks.rados.rados.0.smithi033.stdout:2984: done (2 left) 2024-09-21T10:00:57.638 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 275 2024-09-21T10:00:57.638 INFO:tasks.rados.rados.0.smithi033.stdout:2987: seq_num 941 ranges {0=565248,565248=475136,1040384=327680} 2024-09-21T10:00:57.641 INFO:tasks.rados.rados.0.smithi033.stdout:2987: writing smithi03385152-40 from 0 to 565248 tid 1 2024-09-21T10:00:57.644 INFO:tasks.rados.rados.0.smithi033.stdout:2987: writing smithi03385152-40 from 565248 to 1040384 tid 2 2024-09-21T10:00:57.647 INFO:tasks.rados.rados.0.smithi033.stdout:2987: writing smithi03385152-40 from 1040384 to 1368064 tid 3 2024-09-21T10:00:57.647 INFO:tasks.rados.rados.0.smithi033.stdout:2988: snap_remove snap 270 2024-09-21T10:00:57.951 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:57 smithi033 ceph-mon[98546]: pgmap v570: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 108 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 17 op/s 2024-09-21T10:00:57.952 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:57 smithi033 ceph-mon[98546]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T10:00:57.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:57 smithi033 ceph-mon[96568]: pgmap v570: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 108 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 17 op/s 2024-09-21T10:00:57.952 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:57 smithi033 ceph-mon[96568]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T10:00:57.975 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph orch ps' 2024-09-21T10:00:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:57 smithi105 ceph-mon[86649]: pgmap v570: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 108 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.5 MiB/s wr, 17 op/s 2024-09-21T10:00:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:57 smithi105 ceph-mon[86649]: osdmap e627: 8 total, 8 up, 8 in 2024-09-21T10:00:58.271 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:00:58.535 INFO:tasks.rados.rados.0.smithi033.stdout:2988: done (3 left) 2024-09-21T10:00:58.535 INFO:tasks.rados.rados.0.smithi033.stdout:2989: delete oid 2 current snap is 275 2024-09-21T10:00:58.535 INFO:tasks.rados.rados.0.smithi033.stdout:2985: finishing rollback tid 0 to smithi03385152-28 2024-09-21T10:00:58.535 INFO:tasks.rados.rados.0.smithi033.stdout:2985: finishing rollback tid 1 to smithi03385152-28 2024-09-21T10:00:58.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1515 (ObjNum 161 snap 0 seq_num 1) dirty exists 2024-09-21T10:00:58.535 INFO:tasks.rados.rados.0.smithi033.stdout:2985: finishing rollback tid 2 to smithi03385152-28 2024-09-21T10:00:58.535 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 909 (ObjNum 914 snap 265 seq_num 914) dirty exists 2024-09-21T10:00:58.536 INFO:tasks.rados.rados.0.smithi033.stdout:2987: finishing write tid 1 to smithi03385152-40 2024-09-21T10:00:58.536 INFO:tasks.rados.rados.0.smithi033.stdout:2987: finishing write tid 2 to smithi03385152-40 2024-09-21T10:00:58.536 INFO:tasks.rados.rados.0.smithi033.stdout:2987: finishing write tid 3 to smithi03385152-40 2024-09-21T10:00:58.536 INFO:tasks.rados.rados.0.smithi033.stdout:2987: finishing write tid 4 to smithi03385152-40 2024-09-21T10:00:58.536 INFO:tasks.rados.rados.0.smithi033.stdout:2987: finishing write tid 5 to smithi03385152-40 2024-09-21T10:00:58.536 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1790 (ObjNum 941 snap 275 seq_num 941) dirty exists 2024-09-21T10:00:58.536 INFO:tasks.rados.rados.0.smithi033.stdout:2987: left oid 40 (ObjNum 941 snap 275 seq_num 941) 2024-09-21T10:00:58.538 INFO:tasks.rados.rados.0.smithi033.stdout:2985: done (3 left) 2024-09-21T10:00:58.538 INFO:tasks.rados.rados.0.smithi033.stdout:2986: done (2 left) 2024-09-21T10:00:58.538 INFO:tasks.rados.rados.0.smithi033.stdout:2987: done (1 left) 2024-09-21T10:00:58.539 INFO:tasks.rados.rados.0.smithi033.stdout:2989: done (0 left) 2024-09-21T10:00:58.539 INFO:tasks.rados.rados.0.smithi033.stdout:2990: snap_create 2024-09-21T10:00:58.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:58 smithi033 ceph-mon[96568]: from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:58.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:58 smithi033 ceph-mon[96568]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T10:00:58.663 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:58 smithi033 ceph-mon[96568]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T10:00:58.664 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:58 smithi033 ceph-mon[98546]: from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:58.664 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:58 smithi033 ceph-mon[98546]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T10:00:58.664 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:58 smithi033 ceph-mon[98546]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T10:00:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:58 smithi105 ceph-mon[86649]: from='client.44477 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:00:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:58 smithi105 ceph-mon[86649]: osdmap e628: 8 total, 8 up, 8 in 2024-09-21T10:00:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:58 smithi105 ceph-mon[86649]: osdmap e629: 8 total, 8 up, 8 in 2024-09-21T10:00:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2990: done (0 left) 2024-09-21T10:00:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2991: read oid 43 snap 275 2024-09-21T10:00:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2991: expect (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:00:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2992: read oid 22 snap -1 2024-09-21T10:00:59.536 INFO:tasks.rados.rados.0.smithi033.stdout:2992: expect deleted 2024-09-21T10:00:59.537 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 276 2024-09-21T10:00:59.537 INFO:tasks.rados.rados.0.smithi033.stdout:2993: seq_num 943 ranges {57344=598016,655360=688128,1343488=114688} 2024-09-21T10:00:59.539 INFO:tasks.rados.rados.0.smithi033.stdout:2993: writing smithi03385152-32 from 57344 to 655360 tid 1 2024-09-21T10:00:59.544 INFO:tasks.rados.rados.0.smithi033.stdout:2993: writing smithi03385152-32 from 655360 to 1343488 tid 2 2024-09-21T10:00:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2993: writing smithi03385152-32 from 1343488 to 1458176 tid 3 2024-09-21T10:00:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2994: snap_create 2024-09-21T10:00:59.545 INFO:tasks.rados.rados.0.smithi033.stdout:2993: finishing write tid 1 to smithi03385152-32 2024-09-21T10:00:59.548 INFO:tasks.rados.rados.0.smithi033.stdout:2993: finishing write tid 2 to smithi03385152-32 2024-09-21T10:00:59.549 INFO:tasks.rados.rados.0.smithi033.stdout:2993: finishing write tid 3 to smithi03385152-32 2024-09-21T10:00:59.549 INFO:tasks.rados.rados.0.smithi033.stdout:2993: finishing write tid 4 to smithi03385152-32 2024-09-21T10:00:59.550 INFO:tasks.rados.rados.0.smithi033.stdout:2993: finishing write tid 5 to smithi03385152-32 2024-09-21T10:00:59.550 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1586 (ObjNum 943 snap 276 seq_num 943) dirty exists 2024-09-21T10:00:59.550 INFO:tasks.rados.rados.0.smithi033.stdout:2993: left oid 32 (ObjNum 943 snap 276 seq_num 943) 2024-09-21T10:00:59.570 INFO:tasks.rados.rados.0.smithi033.stdout:2991: expect (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:00:59.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:59 smithi033 ceph-mon[98546]: pgmap v573: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 110 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-21T10:00:59.929 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:00:59 smithi033 ceph-mon[98546]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T10:00:59.929 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:59 smithi033 ceph-mon[96568]: pgmap v573: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 110 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-21T10:00:59.929 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:00:59 smithi033 ceph-mon[96568]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T10:00:59.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:59 smithi105 ceph-mon[86649]: pgmap v573: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 110 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.1 MiB/s wr, 13 op/s 2024-09-21T10:00:59.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:00:59 smithi105 ceph-mon[86649]: osdmap e630: 8 total, 8 up, 8 in 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:alertmanager.a smithi033 *:9093,9094 running (7m) 57s ago 12m 19.6M - 0.27.0 11f11916f8cd 9ab68da7db7a 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:grafana.a smithi105 *:3000 running (19s) 14s ago 12m 48.7M - 10.4.8 5aad1d7cf004 37486b9e0f8e 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:mds.a smithi033 running (61s) 57s ago 13m 11.9M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9bc95e6ef3d0 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:mds.b smithi105 running (42s) 14s ago 13m 15.4M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1ceb42a1686b 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.x smithi105 *:8443,9283,8765 running (6m) 14s ago 17m 457M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5f55827751a5 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:mgr.y smithi033 *:8443,9283,8765 running (7m) 57s ago 18m 528M - 19.3.0-5092-g62da54b4 a8ebb3d62dd1 917c670c2752 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:mon.a smithi033 running (6m) 57s ago 19m 67.8M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 652442dcd6a4 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:mon.b smithi105 running (5m) 14s ago 17m 57.4M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 9f90671a2e29 2024-09-21T10:01:00.367 INFO:teuthology.orchestra.run.smithi033.stdout:mon.c smithi033 running (6m) 57s ago 17m 57.1M 2048M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 5c9071ecac42 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.a smithi033 *:9100 running (7m) 57s ago 11m 12.0M - 1.7.0 72c9c2088986 cce524477a32 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:node-exporter.b smithi105 *:9100 running (7m) 14s ago 11m 10.9M - 1.7.0 72c9c2088986 c79287d80715 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:osd.0 smithi033 running (5m) 57s ago 16m 283M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 28dc5f57b5bc 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:osd.1 smithi033 running (4m) 57s ago 16m 240M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 1deebc8df60c 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:osd.2 smithi033 running (4m) 57s ago 16m 154M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 76dbf29565ba 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:osd.3 smithi033 running (3m) 57s ago 15m 83.1M 2481M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 b85b61817088 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:osd.4 smithi105 running (3m) 14s ago 15m 174M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 01f4f66dbe72 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:osd.5 smithi105 running (2m) 14s ago 14m 209M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 4ff1257a8be1 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:osd.6 smithi105 running (112s) 14s ago 14m 106M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 132bc834f24d 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:osd.7 smithi105 running (77s) 14s ago 14m 89.8M 2480M 19.3.0-5092-g62da54b4 a8ebb3d62dd1 ea10ccd8e9ee 2024-09-21T10:01:00.368 INFO:teuthology.orchestra.run.smithi033.stdout:prometheus.a smithi105 *:9095 running (6m) 14s ago 12m 58.1M - 2.51.0 1d3b7f56885b 670cdb3edd9d 2024-09-21T10:01:00.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:01:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:01:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:00.535 INFO:tasks.rados.rados.0.smithi033.stdout:2991: done (3 left) 2024-09-21T10:01:00.535 INFO:tasks.rados.rados.0.smithi033.stdout:2992: done (2 left) 2024-09-21T10:01:00.535 INFO:tasks.rados.rados.0.smithi033.stdout:2993: done (1 left) 2024-09-21T10:01:00.535 INFO:tasks.rados.rados.0.smithi033.stdout:2994: done (0 left) 2024-09-21T10:01:00.535 INFO:tasks.rados.rados.0.smithi033.stdout:2995: snap_create 2024-09-21T10:01:01.049 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph versions' 2024-09-21T10:01:01.304 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:01.536 INFO:tasks.rados.rados.0.smithi033.stdout:2995: done (0 left) 2024-09-21T10:01:01.537 INFO:tasks.rados.rados.0.smithi033.stdout:2996: delete oid 14 current snap is 278 2024-09-21T10:01:01.544 INFO:tasks.rados.rados.0.smithi033.stdout:2996: done (0 left) 2024-09-21T10:01:01.545 INFO:tasks.rados.rados.0.smithi033.stdout:2997: snap_create 2024-09-21T10:01:01.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-mon[96568]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:01.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-mon[96568]: pgmap v576: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 102 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-21T10:01:01.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-mon[96568]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T10:01:01.712 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:01.712 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:01:01] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-09-21T10:01:01.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-mon[98546]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:01.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-mon[98546]: pgmap v576: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 102 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-21T10:01:01.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-mon[98546]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T10:01:01.713 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:01 smithi105 ceph-mon[86649]: from='client.34569 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-21T10:01:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:01 smithi105 ceph-mon[86649]: pgmap v576: 121 pgs: 2 active+clean+snaptrim, 119 active+clean; 102 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-09-21T10:01:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:01 smithi105 ceph-mon[86649]: osdmap e631: 8 total, 8 up, 8 in 2024-09-21T10:01:01.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:2997: done (0 left) 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:2998: copy_from oid 50 from oid 8 current snap is 279 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:2999: copy_from oid 29 from oid 43 current snap is 279 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:3000: read oid 40 snap -1 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:3000: expect (ObjNum 941 snap 275 seq_num 941) 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:3001: read oid 20 snap -1 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:3001: expect deleted 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:3002: read oid 26 snap -1 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:3002: expect (ObjNum 929 snap 272 seq_num 929) 2024-09-21T10:01:02.547 INFO:tasks.rados.rados.0.smithi033.stdout:3003: snap_remove snap 277 2024-09-21T10:01:02.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:02 smithi033 ceph-mon[98546]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T10:01:02.740 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:02 smithi033 ceph-mon[96568]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T10:01:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:02 smithi105 ceph-mon[86649]: osdmap e632: 8 total, 8 up, 8 in 2024-09-21T10:01:03.204 INFO:teuthology.orchestra.run.smithi033.stdout:{ 2024-09-21T10:01:03.204 INFO:teuthology.orchestra.run.smithi033.stdout: "mon": { 2024-09-21T10:01:03.204 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 3 2024-09-21T10:01:03.204 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T10:01:03.204 INFO:teuthology.orchestra.run.smithi033.stdout: "mgr": { 2024-09-21T10:01:03.204 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: "osd": { 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 8 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: "mds": { 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 2 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: }, 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: "overall": { 2024-09-21T10:01:03.205 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)": 15 2024-09-21T10:01:03.206 INFO:teuthology.orchestra.run.smithi033.stdout: } 2024-09-21T10:01:03.206 INFO:teuthology.orchestra.run.smithi033.stdout:} 2024-09-21T10:01:03.538 INFO:tasks.rados.rados.0.smithi033.stdout:3003: done (5 left) 2024-09-21T10:01:03.538 INFO:tasks.rados.rados.0.smithi033.stdout:3004: delete oid 9 current snap is 279 2024-09-21T10:01:03.538 INFO:tasks.rados.rados.0.smithi033.stdout:3000: expect (ObjNum 941 snap 275 seq_num 941) 2024-09-21T10:01:03.578 INFO:tasks.rados.rados.0.smithi033.stdout:2998: finishing copy_from racing read to smithi03385152-50 2024-09-21T10:01:03.578 INFO:tasks.rados.rados.0.smithi033.stdout:2998: finishing copy_from to smithi03385152-50 2024-09-21T10:01:03.578 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 1494 (ObjNum 920 snap 268 seq_num 920) dirty exists 2024-09-21T10:01:03.578 INFO:tasks.rados.rados.0.smithi033.stdout:3002: expect (ObjNum 929 snap 272 seq_num 929) 2024-09-21T10:01:03.769 INFO:tasks.rados.rados.0.smithi033.stdout:2999: finishing copy_from racing read to smithi03385152-29 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:2999: finishing copy_from to smithi03385152-29 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1512 (ObjNum 909 snap 263 seq_num 909) dirty exists 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:2998: done (5 left) 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:2999: done (4 left) 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:3000: done (3 left) 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:3001: done (2 left) 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:3002: done (1 left) 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:3004: done (0 left) 2024-09-21T10:01:03.770 INFO:tasks.rados.rados.0.smithi033.stdout:3005: snap_remove snap 274 2024-09-21T10:01:03.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:03 smithi033 ceph-mon[98546]: pgmap v579: 121 pgs: 121 active+clean; 94 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T10:01:03.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:03 smithi033 ceph-mon[98546]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T10:01:03.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:03 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/789857161' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:03.866 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:03 smithi033 ceph-mon[98546]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T10:01:03.866 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:03 smithi033 ceph-mon[96568]: pgmap v579: 121 pgs: 121 active+clean; 94 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T10:01:03.866 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:03 smithi033 ceph-mon[96568]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T10:01:03.866 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:03 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/789857161' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:03.867 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:03 smithi033 ceph-mon[96568]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T10:01:03.890 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-09-21T10:01:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:03 smithi105 ceph-mon[86649]: pgmap v579: 121 pgs: 121 active+clean; 94 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 1.3 MiB/s wr, 12 op/s 2024-09-21T10:01:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:03 smithi105 ceph-mon[86649]: osdmap e633: 8 total, 8 up, 8 in 2024-09-21T10:01:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:03 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/789857161' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:03 smithi105 ceph-mon[86649]: osdmap e634: 8 total, 8 up, 8 in 2024-09-21T10:01:04.152 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:04.552 INFO:tasks.rados.rados.0.smithi033.stdout:3005: done (0 left) 2024-09-21T10:01:04.553 INFO:tasks.rados.rados.0.smithi033.stdout:3006: read oid 45 snap 271 2024-09-21T10:01:04.553 INFO:tasks.rados.rados.0.smithi033.stdout:3006: expect (ObjNum 914 snap 265 seq_num 914) 2024-09-21T10:01:04.553 INFO:tasks.rados.rados.0.smithi033.stdout:3007: setattr oid 8 current snap is 279 2024-09-21T10:01:04.553 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 279 2024-09-21T10:01:04.553 INFO:tasks.rados.rados.0.smithi033.stdout:3008: seq_num 946 ranges {0=647168,647168=745472,1392640=57344} 2024-09-21T10:01:04.558 INFO:tasks.rados.rados.0.smithi033.stdout:3008: writing smithi03385152-49 from 0 to 647168 tid 1 2024-09-21T10:01:04.562 INFO:tasks.rados.rados.0.smithi033.stdout:3008: writing smithi03385152-49 from 647168 to 1392640 tid 2 2024-09-21T10:01:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:3008: writing smithi03385152-49 from 1392640 to 1449984 tid 3 2024-09-21T10:01:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 279 2024-09-21T10:01:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 1495 (ObjNum 920 snap 268 seq_num 920) dirty exists 2024-09-21T10:01:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:3008: finishing write tid 1 to smithi03385152-49 2024-09-21T10:01:04.563 INFO:tasks.rados.rados.0.smithi033.stdout:3009: seq_num 947 ranges {0=663552,663552=630784,1294336=172032} 2024-09-21T10:01:04.567 INFO:tasks.rados.rados.0.smithi033.stdout:3009: writing smithi03385152-17 from 0 to 663552 tid 1 2024-09-21T10:01:04.570 INFO:tasks.rados.rados.0.smithi033.stdout:3009: writing smithi03385152-17 from 663552 to 1294336 tid 2 2024-09-21T10:01:04.571 INFO:tasks.rados.rados.0.smithi033.stdout:3009: writing smithi03385152-17 from 1294336 to 1466368 tid 3 2024-09-21T10:01:04.571 INFO:tasks.rados.rados.0.smithi033.stdout:3007: done (3 left) 2024-09-21T10:01:04.571 INFO:tasks.rados.rados.0.smithi033.stdout:3010: snap_remove snap 273 2024-09-21T10:01:05.560 INFO:tasks.rados.rados.0.smithi033.stdout:3010: done (3 left) 2024-09-21T10:01:05.560 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 279 2024-09-21T10:01:05.560 INFO:tasks.rados.rados.0.smithi033.stdout:3011: seq_num 948 ranges {1253376=679936,1933312=516096,2449408=286720} 2024-09-21T10:01:05.570 INFO:tasks.rados.rados.0.smithi033.stdout:3011: writing smithi03385152-47 from 1253376 to 1933312 tid 1 2024-09-21T10:01:05.573 INFO:tasks.rados.rados.0.smithi033.stdout:3011: writing smithi03385152-47 from 1933312 to 2449408 tid 2 2024-09-21T10:01:05.576 INFO:tasks.rados.rados.0.smithi033.stdout:3011: writing smithi03385152-47 from 2449408 to 2736128 tid 3 2024-09-21T10:01:05.576 INFO:tasks.rados.rados.0.smithi033.stdout:3012: rollback oid 46 current snap is 279 2024-09-21T10:01:05.576 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 46 to 275 2024-09-21T10:01:05.576 INFO:tasks.rados.rados.0.smithi033.stdout:3008: finishing write tid 2 to smithi03385152-49 2024-09-21T10:01:05.576 INFO:tasks.rados.rados.0.smithi033.stdout:3006: expect (ObjNum 914 snap 265 seq_num 914) 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3008: finishing write tid 3 to smithi03385152-49 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3008: finishing write tid 4 to smithi03385152-49 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3009: finishing write tid 1 to smithi03385152-17 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3008: finishing write tid 5 to smithi03385152-49 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1524 (ObjNum 946 snap 279 seq_num 946) dirty exists 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3008: left oid 49 (ObjNum 946 snap 279 seq_num 946) 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3009: finishing write tid 2 to smithi03385152-17 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3009: finishing write tid 3 to smithi03385152-17 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3006: done (4 left) 2024-09-21T10:01:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3008: done (3 left) 2024-09-21T10:01:05.605 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 279 2024-09-21T10:01:05.605 INFO:tasks.rados.rados.0.smithi033.stdout:3009: finishing write tid 4 to smithi03385152-17 2024-09-21T10:01:05.605 INFO:tasks.rados.rados.0.smithi033.stdout:3013: seq_num 949 ranges {9363456=704512,10067968=794624,10862592=8192} 2024-09-21T10:01:05.659 INFO:tasks.rados.rados.0.smithi033.stdout:3013: writing smithi03385152-43 from 9363456 to 10067968 tid 1 2024-09-21T10:01:05.664 INFO:tasks.rados.rados.0.smithi033.stdout:3013: writing smithi03385152-43 from 10067968 to 10862592 tid 2 2024-09-21T10:01:05.664 INFO:tasks.rados.rados.0.smithi033.stdout:3013: writing smithi03385152-43 from 10862592 to 10870784 tid 3 2024-09-21T10:01:05.664 INFO:tasks.rados.rados.0.smithi033.stdout:3014: read oid 6 snap 271 2024-09-21T10:01:05.664 INFO:tasks.rados.rados.0.smithi033.stdout:3014: expect deleted 2024-09-21T10:01:05.664 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 279 2024-09-21T10:01:05.664 INFO:tasks.rados.rados.0.smithi033.stdout:3015: seq_num 950 ranges {917504=720896,1638400=679936,2318336=122880} 2024-09-21T10:01:05.673 INFO:tasks.rados.rados.0.smithi033.stdout:3015: writing smithi03385152-42 from 917504 to 1638400 tid 1 2024-09-21T10:01:05.678 INFO:tasks.rados.rados.0.smithi033.stdout:3015: writing smithi03385152-42 from 1638400 to 2318336 tid 2 2024-09-21T10:01:05.679 INFO:tasks.rados.rados.0.smithi033.stdout:3015: writing smithi03385152-42 from 2318336 to 2441216 tid 3 2024-09-21T10:01:05.679 INFO:tasks.rados.rados.0.smithi033.stdout:3016: copy_from oid 5 from oid 9 current snap is 279 2024-09-21T10:01:05.679 INFO:tasks.rados.rados.0.smithi033.stdout:3009: finishing write tid 5 to smithi03385152-17 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 1528 (ObjNum 947 snap 279 seq_num 947) dirty exists 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3009: left oid 17 (ObjNum 947 snap 279 seq_num 947) 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3011: finishing write tid 1 to smithi03385152-47 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3009: done (6 left) 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3017: delete oid 19 current snap is 279 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3011: finishing write tid 2 to smithi03385152-47 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3011: finishing write tid 3 to smithi03385152-47 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3011: finishing write tid 4 to smithi03385152-47 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3011: finishing write tid 5 to smithi03385152-47 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1502 (ObjNum 948 snap 279 seq_num 948) dirty exists 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3011: left oid 47 (ObjNum 948 snap 279 seq_num 948) 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3012: finishing rollback tid 0 to smithi03385152-46 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3012: finishing rollback tid 1 to smithi03385152-46 2024-09-21T10:01:05.680 INFO:tasks.rados.rados.0.smithi033.stdout:3012: finishing rollback tid 2 to smithi03385152-46 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1812 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:3013: finishing write tid 1 to smithi03385152-43 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:3013: finishing write tid 2 to smithi03385152-43 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:3013: finishing write tid 3 to smithi03385152-43 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:3013: finishing write tid 4 to smithi03385152-43 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:3013: finishing write tid 5 to smithi03385152-43 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1519 (ObjNum 949 snap 279 seq_num 949) dirty exists 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:3013: left oid 43 (ObjNum 949 snap 279 seq_num 949) 2024-09-21T10:01:05.681 INFO:tasks.rados.rados.0.smithi033.stdout:3015: finishing write tid 1 to smithi03385152-42 2024-09-21T10:01:05.682 INFO:tasks.rados.rados.0.smithi033.stdout:3011: done (6 left) 2024-09-21T10:01:05.682 INFO:tasks.rados.rados.0.smithi033.stdout:3012: done (5 left) 2024-09-21T10:01:05.682 INFO:tasks.rados.rados.0.smithi033.stdout:3013: done (4 left) 2024-09-21T10:01:05.682 INFO:tasks.rados.rados.0.smithi033.stdout:3014: done (3 left) 2024-09-21T10:01:05.682 INFO:tasks.rados.rados.0.smithi033.stdout:3017: done (2 left) 2024-09-21T10:01:05.683 INFO:tasks.rados.rados.0.smithi033.stdout:3018: read oid 48 snap -1 2024-09-21T10:01:05.683 INFO:tasks.rados.rados.0.smithi033.stdout:3018: expect (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:01:05.683 INFO:tasks.rados.rados.0.smithi033.stdout:3019: copy_from oid 40 from oid 26 current snap is 279 2024-09-21T10:01:05.683 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 279 2024-09-21T10:01:05.683 INFO:tasks.rados.rados.0.smithi033.stdout:3020: seq_num 952 ranges {0=753664,753664=499712,1253376=303104} 2024-09-21T10:01:05.686 INFO:tasks.rados.rados.0.smithi033.stdout:3020: writing smithi03385152-14 from 0 to 753664 tid 1 2024-09-21T10:01:05.689 INFO:tasks.rados.rados.0.smithi033.stdout:3020: writing smithi03385152-14 from 753664 to 1253376 tid 2 2024-09-21T10:01:05.691 INFO:tasks.rados.rados.0.smithi033.stdout:3020: writing smithi03385152-14 from 1253376 to 1556480 tid 3 2024-09-21T10:01:05.691 INFO:tasks.rados.rados.0.smithi033.stdout:3021: read oid 41 snap -1 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3021: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3022: read oid 36 snap -1 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3022: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3023: snap_create 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3015: finishing write tid 2 to smithi03385152-42 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3016: finishing copy_from to smithi03385152-5 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3016: got expected ENOENT (src dne) 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3016: finishing copy_from racing read to smithi03385152-5 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3015: finishing write tid 3 to smithi03385152-42 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3015: finishing write tid 4 to smithi03385152-42 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3015: finishing write tid 5 to smithi03385152-42 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 1795 (ObjNum 950 snap 279 seq_num 950) dirty exists 2024-09-21T10:01:05.692 INFO:tasks.rados.rados.0.smithi033.stdout:3015: left oid 42 (ObjNum 950 snap 279 seq_num 950) 2024-09-21T10:01:05.699 INFO:tasks.rados.rados.0.smithi033.stdout:3019: finishing copy_from racing read to smithi03385152-40 2024-09-21T10:01:05.700 INFO:tasks.rados.rados.0.smithi033.stdout:3021: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:05.702 INFO:tasks.rados.rados.0.smithi033.stdout:3019: finishing copy_from to smithi03385152-40 2024-09-21T10:01:05.702 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1520 (ObjNum 929 snap 272 seq_num 929) dirty exists 2024-09-21T10:01:05.720 INFO:tasks.rados.rados.0.smithi033.stdout:3020: finishing write tid 1 to smithi03385152-14 2024-09-21T10:01:05.720 INFO:tasks.rados.rados.0.smithi033.stdout:3020: finishing write tid 2 to smithi03385152-14 2024-09-21T10:01:05.728 INFO:tasks.rados.rados.0.smithi033.stdout:3018: expect (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:01:05.794 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:05 smithi033 ceph-mon[98546]: pgmap v582: 121 pgs: 121 active+clean; 87 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 672 KiB/s wr, 10 op/s 2024-09-21T10:01:05.794 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:05 smithi033 ceph-mon[98546]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T10:01:05.794 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:05 smithi033 ceph-mon[96568]: pgmap v582: 121 pgs: 121 active+clean; 87 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 672 KiB/s wr, 10 op/s 2024-09-21T10:01:05.794 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:05 smithi033 ceph-mon[96568]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T10:01:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:05 smithi105 ceph-mon[86649]: pgmap v582: 121 pgs: 121 active+clean; 87 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 672 KiB/s wr, 10 op/s 2024-09-21T10:01:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:05 smithi105 ceph-mon[86649]: osdmap e635: 8 total, 8 up, 8 in 2024-09-21T10:01:06.162 INFO:teuthology.orchestra.run.smithi033.stdout:true 2024-09-21T10:01:06.527 INFO:tasks.rados.rados.0.smithi033.stdout:3020: finishing write tid 3 to smithi03385152-14 2024-09-21T10:01:06.527 INFO:tasks.rados.rados.0.smithi033.stdout:3020: finishing write tid 4 to smithi03385152-14 2024-09-21T10:01:06.527 INFO:tasks.rados.rados.0.smithi033.stdout:3022: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:01:06.529 INFO:tasks.rados.rados.0.smithi033.stdout:3020: finishing write tid 5 to smithi03385152-14 2024-09-21T10:01:06.529 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1816 (ObjNum 952 snap 279 seq_num 952) dirty exists 2024-09-21T10:01:06.529 INFO:tasks.rados.rados.0.smithi033.stdout:3020: left oid 14 (ObjNum 952 snap 279 seq_num 952) 2024-09-21T10:01:06.563 INFO:tasks.rados.rados.0.smithi033.stdout:3015: done (7 left) 2024-09-21T10:01:06.563 INFO:tasks.rados.rados.0.smithi033.stdout:3016: done (6 left) 2024-09-21T10:01:06.563 INFO:tasks.rados.rados.0.smithi033.stdout:3018: done (5 left) 2024-09-21T10:01:06.563 INFO:tasks.rados.rados.0.smithi033.stdout:3019: done (4 left) 2024-09-21T10:01:06.563 INFO:tasks.rados.rados.0.smithi033.stdout:3020: done (3 left) 2024-09-21T10:01:06.563 INFO:tasks.rados.rados.0.smithi033.stdout:3021: done (2 left) 2024-09-21T10:01:06.563 INFO:tasks.rados.rados.0.smithi033.stdout:3022: done (1 left) 2024-09-21T10:01:06.564 INFO:tasks.rados.rados.0.smithi033.stdout:3023: done (0 left) 2024-09-21T10:01:06.564 INFO:tasks.rados.rados.0.smithi033.stdout:3024: delete oid 12 current snap is 280 2024-09-21T10:01:06.568 INFO:tasks.rados.rados.0.smithi033.stdout:3024: done (0 left) 2024-09-21T10:01:06.568 INFO:tasks.rados.rados.0.smithi033.stdout:3025: read oid 38 snap -1 2024-09-21T10:01:06.568 INFO:tasks.rados.rados.0.smithi033.stdout:3025: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:01:06.568 INFO:tasks.rados.rados.0.smithi033.stdout:3026: setattr oid 49 current snap is 280 2024-09-21T10:01:06.568 INFO:tasks.rados.rados.0.smithi033.stdout:3025: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:01:06.571 INFO:tasks.rados.rados.0.smithi033.stdout:3025: done (1 left) 2024-09-21T10:01:06.571 INFO:tasks.rados.rados.0.smithi033.stdout:3027: rollback oid 10 current snap is 280 2024-09-21T10:01:06.571 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 10 to 279 2024-09-21T10:01:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:3028: snap_create 2024-09-21T10:01:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:3027: finishing rollback tid 1 to smithi03385152-10 2024-09-21T10:01:06.572 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 0 (ObjNum 65 snap 0 seq_num 1431330924) dirty dne 2024-09-21T10:01:06.574 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1530 (ObjNum 946 snap 279 seq_num 946) dirty exists 2024-09-21T10:01:06.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:06 smithi033 ceph-mon[98546]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T10:01:06.739 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:06 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1996207622' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:06.739 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:06 smithi033 ceph-mon[96568]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T10:01:06.739 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:06 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1996207622' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:06.763 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -e sha1=62da54b4b83a96ca9976b7e1d0c8272a564a2208 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-09-21T10:01:07.018 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:01:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:06 smithi105 ceph-mon[86649]: osdmap e636: 8 total, 8 up, 8 in 2024-09-21T10:01:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:06 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1996207622' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:07.564 INFO:tasks.rados.rados.0.smithi033.stdout:3026: done (2 left) 2024-09-21T10:01:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:3027: done (1 left) 2024-09-21T10:01:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:3028: done (0 left) 2024-09-21T10:01:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:3029: rollback oid 30 current snap is 281 2024-09-21T10:01:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 30 to 280 2024-09-21T10:01:07.565 INFO:tasks.rados.rados.0.smithi033.stdout:3030: snap_remove snap 276 2024-09-21T10:01:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:07 smithi033 ceph-mon[98546]: pgmap v585: 121 pgs: 121 active+clean; 98 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.0 MiB/s wr, 6 op/s 2024-09-21T10:01:07.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:07 smithi033 ceph-mon[98546]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T10:01:07.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:07 smithi033 ceph-mon[96568]: pgmap v585: 121 pgs: 121 active+clean; 98 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.0 MiB/s wr, 6 op/s 2024-09-21T10:01:07.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:07 smithi033 ceph-mon[96568]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T10:01:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:07 smithi105 ceph-mon[86649]: pgmap v585: 121 pgs: 121 active+clean; 98 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 3.0 MiB/s wr, 6 op/s 2024-09-21T10:01:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:07 smithi105 ceph-mon[86649]: osdmap e637: 8 total, 8 up, 8 in 2024-09-21T10:01:08.546 INFO:tasks.rados.rados.0.smithi033.stdout:3030: done (1 left) 2024-09-21T10:01:08.546 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 281 2024-09-21T10:01:08.547 INFO:tasks.rados.rados.0.smithi033.stdout:3031: seq_num 954 ranges {1449984=786432,2236416=671744,2908160=131072} 2024-09-21T10:01:08.559 INFO:tasks.rados.rados.0.smithi033.stdout:3031: writing smithi03385152-49 from 1449984 to 2236416 tid 1 2024-09-21T10:01:08.563 INFO:tasks.rados.rados.0.smithi033.stdout:3031: writing smithi03385152-49 from 2236416 to 2908160 tid 2 2024-09-21T10:01:08.564 INFO:tasks.rados.rados.0.smithi033.stdout:3031: writing smithi03385152-49 from 2908160 to 3039232 tid 3 2024-09-21T10:01:08.564 INFO:tasks.rados.rados.0.smithi033.stdout:3032: snap_remove snap 278 2024-09-21T10:01:08.741 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:08 smithi033 ceph-mon[98546]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T10:01:08.741 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:08 smithi033 ceph-mon[98546]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T10:01:08.741 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:08 smithi033 ceph-mon[96568]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T10:01:08.741 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:08 smithi033 ceph-mon[96568]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T10:01:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:08 smithi105 ceph-mon[86649]: osdmap e638: 8 total, 8 up, 8 in 2024-09-21T10:01:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:08 smithi105 ceph-mon[86649]: osdmap e639: 8 total, 8 up, 8 in 2024-09-21T10:01:09.032 INFO:teuthology.orchestra.run.smithi033.stdout: "ceph version 19.3.0-5092-g62da54b4 (62da54b4b83a96ca9976b7e1d0c8272a564a2208) squid (dev)" 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3032: done (2 left) 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3033: copy_from oid 12 from oid 33 current snap is 281 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3029: finishing rollback tid 0 to smithi03385152-30 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3029: finishing rollback tid 1 to smithi03385152-30 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3029: finishing rollback tid 2 to smithi03385152-30 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2242 (ObjNum 811 snap 237 seq_num 811) dirty exists 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3031: finishing write tid 1 to smithi03385152-49 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3031: finishing write tid 2 to smithi03385152-49 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3031: finishing write tid 3 to smithi03385152-49 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3031: finishing write tid 4 to smithi03385152-49 2024-09-21T10:01:09.539 INFO:tasks.rados.rados.0.smithi033.stdout:3031: finishing write tid 5 to smithi03385152-49 2024-09-21T10:01:09.540 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1534 (ObjNum 954 snap 281 seq_num 954) dirty exists 2024-09-21T10:01:09.540 INFO:tasks.rados.rados.0.smithi033.stdout:3031: left oid 49 (ObjNum 954 snap 281 seq_num 954) 2024-09-21T10:01:09.540 INFO:tasks.rados.rados.0.smithi033.stdout:3029: done (2 left) 2024-09-21T10:01:09.540 INFO:tasks.rados.rados.0.smithi033.stdout:3031: done (1 left) 2024-09-21T10:01:09.540 INFO:tasks.rados.rados.0.smithi033.stdout:3034: delete oid 47 current snap is 281 2024-09-21T10:01:09.544 INFO:tasks.rados.rados.0.smithi033.stdout:3034: done (1 left) 2024-09-21T10:01:09.545 INFO:tasks.rados.rados.0.smithi033.stdout:3035: copy_from oid 30 from oid 39 current snap is 281 2024-09-21T10:01:09.545 INFO:tasks.rados.rados.0.smithi033.stdout:3036: setattr oid 10 current snap is 281 2024-09-21T10:01:09.546 INFO:tasks.rados.rados.0.smithi033.stdout:3037: rollback oid 21 current snap is 281 2024-09-21T10:01:09.546 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 271 2024-09-21T10:01:09.546 INFO:tasks.rados.rados.0.smithi033.stdout:3038: rmattr oid 24 current snap is 281 2024-09-21T10:01:09.546 INFO:tasks.rados.rados.0.smithi033.stdout:3038: done (4 left) 2024-09-21T10:01:09.546 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 281 2024-09-21T10:01:09.547 INFO:tasks.rados.rados.0.smithi033.stdout:3039: seq_num 956 ranges {1351680=417792,1769472=434176,2203648=770048} 2024-09-21T10:01:09.556 INFO:tasks.rados.rados.0.smithi033.stdout:3039: writing smithi03385152-13 from 1351680 to 1769472 tid 1 2024-09-21T10:01:09.559 INFO:tasks.rados.rados.0.smithi033.stdout:3039: writing smithi03385152-13 from 1769472 to 2203648 tid 2 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:3039: writing smithi03385152-13 from 2203648 to 2973696 tid 3 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:3040: delete oid 4 current snap is 281 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:3035: finishing copy_from to smithi03385152-30 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:3035: got expected ENOENT (src dne) 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:3035: finishing copy_from racing read to smithi03385152-30 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:3033: finishing copy_from racing read to smithi03385152-12 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 2176 (ObjNum 49 snap 0 seq_num 1533112144) dirty exists 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:3037: finishing rollback tid 0 to smithi03385152-21 2024-09-21T10:01:09.564 INFO:tasks.rados.rados.0.smithi033.stdout:3037: finishing rollback tid 1 to smithi03385152-21 2024-09-21T10:01:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1517 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T10:01:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:3033: finishing copy_from to smithi03385152-12 2024-09-21T10:01:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1616 (ObjNum 919 snap 268 seq_num 919) dirty exists 2024-09-21T10:01:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:3039: finishing write tid 1 to smithi03385152-13 2024-09-21T10:01:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:3039: finishing write tid 2 to smithi03385152-13 2024-09-21T10:01:09.565 INFO:tasks.rados.rados.0.smithi033.stdout:3033: done (5 left) 2024-09-21T10:01:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:3035: done (4 left) 2024-09-21T10:01:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:3036: done (3 left) 2024-09-21T10:01:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:3037: done (2 left) 2024-09-21T10:01:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:3040: done (1 left) 2024-09-21T10:01:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:3041: copy_from oid 23 from oid 33 current snap is 281 2024-09-21T10:01:09.566 INFO:tasks.rados.rados.0.smithi033.stdout:3042: delete oid 40 current snap is 281 2024-09-21T10:01:09.568 INFO:tasks.rados.rados.0.smithi033.stdout:3042: done (2 left) 2024-09-21T10:01:09.568 INFO:tasks.rados.rados.0.smithi033.stdout:3043: setattr oid 31 current snap is 281 2024-09-21T10:01:09.568 INFO:tasks.rados.rados.0.smithi033.stdout:3039: finishing write tid 3 to smithi03385152-13 2024-09-21T10:01:09.568 INFO:tasks.rados.rados.0.smithi033.stdout:3039: finishing write tid 4 to smithi03385152-13 2024-09-21T10:01:09.570 INFO:tasks.rados.rados.0.smithi033.stdout:3044: rollback oid 28 current snap is 281 2024-09-21T10:01:09.570 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 279 2024-09-21T10:01:09.570 INFO:tasks.rados.rados.0.smithi033.stdout:3045: copy_from oid 21 from oid 48 current snap is 281 2024-09-21T10:01:09.570 INFO:tasks.rados.rados.0.smithi033.stdout:3046: rollback oid 24 current snap is 281 2024-09-21T10:01:09.570 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 275 2024-09-21T10:01:09.570 INFO:tasks.rados.rados.0.smithi033.stdout:3047: read oid 32 snap -1 2024-09-21T10:01:09.570 INFO:tasks.rados.rados.0.smithi033.stdout:3047: expect (ObjNum 943 snap 276 seq_num 943) 2024-09-21T10:01:09.570 INFO:tasks.rados.rados.0.smithi033.stdout:3048: delete oid 46 current snap is 281 2024-09-21T10:01:09.571 INFO:tasks.rados.rados.0.smithi033.stdout:3039: finishing write tid 5 to smithi03385152-13 2024-09-21T10:01:09.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2251 (ObjNum 956 snap 281 seq_num 956) dirty exists 2024-09-21T10:01:09.571 INFO:tasks.rados.rados.0.smithi033.stdout:3039: left oid 13 (ObjNum 956 snap 281 seq_num 956) 2024-09-21T10:01:09.574 INFO:tasks.rados.rados.0.smithi033.stdout:3046: finishing rollback tid 0 to smithi03385152-24 2024-09-21T10:01:09.574 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2252 (ObjNum 33 snap 0 seq_num 91682208) dirty exists 2024-09-21T10:01:09.575 INFO:tasks.rados.rados.0.smithi033.stdout:3041: finishing copy_from racing read to smithi03385152-23 2024-09-21T10:01:09.575 INFO:tasks.rados.rados.0.smithi033.stdout:3039: done (7 left) 2024-09-21T10:01:09.575 INFO:tasks.rados.rados.0.smithi033.stdout:3043: done (6 left) 2024-09-21T10:01:09.575 INFO:tasks.rados.rados.0.smithi033.stdout:3048: done (5 left) 2024-09-21T10:01:09.575 INFO:tasks.rados.rados.0.smithi033.stdout:3049: setattr oid 39 current snap is 281 2024-09-21T10:01:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3044: finishing rollback tid 0 to smithi03385152-28 2024-09-21T10:01:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3046: finishing rollback tid 1 to smithi03385152-24 2024-09-21T10:01:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3046: finishing rollback tid 2 to smithi03385152-24 2024-09-21T10:01:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 2179 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:01:09.578 INFO:tasks.rados.rados.0.smithi033.stdout:3046: done (5 left) 2024-09-21T10:01:09.578 INFO:tasks.rados.rados.0.smithi033.stdout:3050: snap_remove snap 271 2024-09-21T10:01:09.695 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:09 smithi033 ceph-mon[98546]: pgmap v588: 121 pgs: 121 active+clean; 98 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.6 MiB/s wr, 7 op/s 2024-09-21T10:01:09.696 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:09 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3496266074' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:09.696 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:09 smithi033 ceph-mon[98546]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T10:01:09.696 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:09 smithi033 ceph-mon[96568]: pgmap v588: 121 pgs: 121 active+clean; 98 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.6 MiB/s wr, 7 op/s 2024-09-21T10:01:09.696 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:09 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3496266074' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:09.696 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:09 smithi033 ceph-mon[96568]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T10:01:09.719 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-21T10:01:09.720 INFO:teuthology.task.print:**** done end upgrade, wait... 2024-09-21T10:01:09.720 DEBUG:teuthology.parallel:result is None 2024-09-21T10:01:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:09 smithi105 ceph-mon[86649]: pgmap v588: 121 pgs: 121 active+clean; 98 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.6 MiB/s wr, 7 op/s 2024-09-21T10:01:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:09 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3496266074' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-21T10:01:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:09 smithi105 ceph-mon[86649]: osdmap e640: 8 total, 8 up, 8 in 2024-09-21T10:01:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:01:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:01:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:10.582 INFO:tasks.rados.rados.0.smithi033.stdout:3047: expect (ObjNum 943 snap 276 seq_num 943) 2024-09-21T10:01:10.627 INFO:tasks.rados.rados.0.smithi033.stdout:3041: finishing copy_from to smithi03385152-23 2024-09-21T10:01:10.627 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 1616 (ObjNum 919 snap 268 seq_num 919) dirty exists 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3044: finishing rollback tid 1 to smithi03385152-28 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3044: finishing rollback tid 2 to smithi03385152-28 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 912 (ObjNum 914 snap 265 seq_num 914) dirty exists 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1055 (ObjNum 33 snap 0 seq_num 91309024) dirty exists 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3045: finishing copy_from racing read to smithi03385152-21 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3045: finishing copy_from to smithi03385152-21 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1512 (ObjNum 909 snap 263 seq_num 909) dirty exists 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3041: done (5 left) 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3044: done (4 left) 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3045: done (3 left) 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3047: done (2 left) 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3049: done (1 left) 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3050: done (0 left) 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3051: rmattr oid 48 current snap is 281 2024-09-21T10:01:10.628 INFO:tasks.rados.rados.0.smithi033.stdout:3052: read oid 20 snap -1 2024-09-21T10:01:10.629 INFO:tasks.rados.rados.0.smithi033.stdout:3052: expect deleted 2024-09-21T10:01:10.629 INFO:tasks.rados.rados.0.smithi033.stdout:3053: read oid 35 snap -1 2024-09-21T10:01:10.629 INFO:tasks.rados.rados.0.smithi033.stdout:3053: expect deleted 2024-09-21T10:01:10.629 INFO:tasks.rados.rados.0.smithi033.stdout:3054: copy_from oid 10 from oid 31 current snap is 281 2024-09-21T10:01:10.629 INFO:tasks.rados.rados.0.smithi033.stdout:3055: rollback oid 15 current snap is 281 2024-09-21T10:01:10.629 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 15 to 280 2024-09-21T10:01:10.629 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 281 2024-09-21T10:01:10.629 INFO:tasks.rados.rados.0.smithi033.stdout:3056: seq_num 960 ranges {999424=483328,1482752=425984,1908736=761856,2670592=16384} 2024-09-21T10:01:10.636 INFO:tasks.rados.rados.0.smithi033.stdout:3056: writing smithi03385152-12 from 999424 to 1482752 tid 1 2024-09-21T10:01:10.639 INFO:tasks.rados.rados.0.smithi033.stdout:3056: writing smithi03385152-12 from 1482752 to 1908736 tid 2 2024-09-21T10:01:10.643 INFO:tasks.rados.rados.0.smithi033.stdout:3056: writing smithi03385152-12 from 1908736 to 2670592 tid 3 2024-09-21T10:01:10.644 INFO:tasks.rados.rados.0.smithi033.stdout:3056: writing smithi03385152-12 from 2670592 to 2686976 tid 4 2024-09-21T10:01:10.644 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 281 2024-09-21T10:01:10.644 INFO:tasks.rados.rados.0.smithi033.stdout:3057: seq_num 961 ranges {40960=499712,540672=712704,1253376=491520} 2024-09-21T10:01:10.646 INFO:tasks.rados.rados.0.smithi033.stdout:3057: writing smithi03385152-16 from 40960 to 540672 tid 1 2024-09-21T10:01:10.651 INFO:tasks.rados.rados.0.smithi033.stdout:3057: writing smithi03385152-16 from 540672 to 1253376 tid 2 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:3057: writing smithi03385152-16 from 1253376 to 1744896 tid 3 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:3058: rollback oid 38 current snap is 281 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 280 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 1834 (ObjNum 909 snap 263 seq_num 909) dirty exists 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:3055: finishing rollback tid 0 to smithi03385152-15 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:3055: finishing rollback tid 1 to smithi03385152-15 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:3055: finishing rollback tid 2 to smithi03385152-15 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 1498 (ObjNum 912 snap 265 seq_num 912) dirty exists 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:3054: finishing copy_from racing read to smithi03385152-10 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:3054: finishing copy_from to smithi03385152-10 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 2252 (ObjNum 49 snap 0 seq_num 0) dirty exists 2024-09-21T10:01:10.655 INFO:tasks.rados.rados.0.smithi033.stdout:3056: finishing write tid 1 to smithi03385152-12 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3056: finishing write tid 2 to smithi03385152-12 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3051: done (7 left) 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3052: done (6 left) 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3053: done (5 left) 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3054: done (4 left) 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3055: done (3 left) 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3059: copy_from oid 30 from oid 41 current snap is 281 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3056: finishing write tid 3 to smithi03385152-12 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3056: finishing write tid 4 to smithi03385152-12 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3056: finishing write tid 5 to smithi03385152-12 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3057: finishing write tid 1 to smithi03385152-16 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3056: finishing write tid 6 to smithi03385152-12 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 1621 (ObjNum 960 snap 281 seq_num 960) dirty exists 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3056: left oid 12 (ObjNum 960 snap 281 seq_num 960) 2024-09-21T10:01:10.656 INFO:tasks.rados.rados.0.smithi033.stdout:3057: finishing write tid 2 to smithi03385152-16 2024-09-21T10:01:10.657 INFO:tasks.rados.rados.0.smithi033.stdout:3056: done (3 left) 2024-09-21T10:01:10.657 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 281 2024-09-21T10:01:10.657 INFO:tasks.rados.rados.0.smithi033.stdout:3060: seq_num 962 ranges {3063808=516096,3579904=589824,4169728=614400} 2024-09-21T10:01:10.674 INFO:tasks.rados.rados.0.smithi033.stdout:3060: writing smithi03385152-15 from 3063808 to 3579904 tid 1 2024-09-21T10:01:10.677 INFO:tasks.rados.rados.0.smithi033.stdout:3060: writing smithi03385152-15 from 3579904 to 4169728 tid 2 2024-09-21T10:01:10.682 INFO:tasks.rados.rados.0.smithi033.stdout:3060: writing smithi03385152-15 from 4169728 to 4784128 tid 3 2024-09-21T10:01:10.682 INFO:tasks.rados.rados.0.smithi033.stdout:3061: copy_from oid 50 from oid 42 current snap is 281 2024-09-21T10:01:10.682 INFO:tasks.rados.rados.0.smithi033.stdout:3057: finishing write tid 3 to smithi03385152-16 2024-09-21T10:01:10.682 INFO:tasks.rados.rados.0.smithi033.stdout:3057: finishing write tid 4 to smithi03385152-16 2024-09-21T10:01:10.682 INFO:tasks.rados.rados.0.smithi033.stdout:3058: finishing rollback tid 0 to smithi03385152-38 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3058: finishing rollback tid 1 to smithi03385152-38 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3057: finishing write tid 5 to smithi03385152-16 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2265 (ObjNum 961 snap 281 seq_num 961) dirty exists 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3057: left oid 16 (ObjNum 961 snap 281 seq_num 961) 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3058: finishing rollback tid 2 to smithi03385152-38 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 1799 (ObjNum 385 snap 0 seq_num 91894320) dirty exists 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3059: finishing copy_from racing read to smithi03385152-30 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3059: finishing copy_from to smithi03385152-30 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 1786 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3060: finishing write tid 1 to smithi03385152-15 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3060: finishing write tid 2 to smithi03385152-15 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3057: done (4 left) 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3058: done (3 left) 2024-09-21T10:01:10.683 INFO:tasks.rados.rados.0.smithi033.stdout:3059: done (2 left) 2024-09-21T10:01:10.684 INFO:tasks.rados.rados.0.smithi033.stdout:3062: rollback oid 38 current snap is 281 2024-09-21T10:01:10.684 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 275 2024-09-21T10:01:10.684 INFO:tasks.rados.rados.0.smithi033.stdout:3063: copy_from oid 9 from oid 22 current snap is 281 2024-09-21T10:01:10.684 INFO:tasks.rados.rados.0.smithi033.stdout:3064: snap_create 2024-09-21T10:01:10.684 INFO:tasks.rados.rados.0.smithi033.stdout:3060: finishing write tid 3 to smithi03385152-15 2024-09-21T10:01:10.684 INFO:tasks.rados.rados.0.smithi033.stdout:3060: finishing write tid 4 to smithi03385152-15 2024-09-21T10:01:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3062: finishing rollback tid 0 to smithi03385152-38 2024-09-21T10:01:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3060: finishing write tid 5 to smithi03385152-15 2024-09-21T10:01:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 1502 (ObjNum 962 snap 281 seq_num 962) dirty exists 2024-09-21T10:01:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3060: left oid 15 (ObjNum 962 snap 281 seq_num 962) 2024-09-21T10:01:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3062: finishing rollback tid 1 to smithi03385152-38 2024-09-21T10:01:10.686 INFO:tasks.rados.rados.0.smithi033.stdout:3062: finishing rollback tid 2 to smithi03385152-38 2024-09-21T10:01:10.686 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 1802 (ObjNum 385 snap 0 seq_num 91894320) dirty exists 2024-09-21T10:01:10.690 INFO:tasks.rados.rados.0.smithi033.stdout:3063: finishing copy_from to smithi03385152-9 2024-09-21T10:01:10.690 INFO:tasks.rados.rados.0.smithi033.stdout:3063: got expected ENOENT (src dne) 2024-09-21T10:01:10.690 INFO:tasks.rados.rados.0.smithi033.stdout:3063: finishing copy_from racing read to smithi03385152-9 2024-09-21T10:01:10.695 INFO:tasks.rados.rados.0.smithi033.stdout:3061: finishing copy_from racing read to smithi03385152-50 2024-09-21T10:01:10.696 INFO:tasks.rados.rados.0.smithi033.stdout:3061: finishing copy_from to smithi03385152-50 2024-09-21T10:01:10.696 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 1795 (ObjNum 950 snap 279 seq_num 950) dirty exists 2024-09-21T10:01:11.592 INFO:tasks.rados.rados.0.smithi033.stdout:3060: done (4 left) 2024-09-21T10:01:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:3061: done (3 left) 2024-09-21T10:01:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:3062: done (2 left) 2024-09-21T10:01:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:3063: done (1 left) 2024-09-21T10:01:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:3064: done (0 left) 2024-09-21T10:01:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:3065: setattr oid 22 current snap is 282 2024-09-21T10:01:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 282 2024-09-21T10:01:11.593 INFO:tasks.rados.rados.0.smithi033.stdout:3066: seq_num 963 ranges {3776512=532480,4308992=475136,4784128=532480,5316608=196608} 2024-09-21T10:01:11.617 INFO:tasks.rados.rados.0.smithi033.stdout:3066: writing smithi03385152-26 from 3776512 to 4308992 tid 1 2024-09-21T10:01:11.619 INFO:tasks.rados.rados.0.smithi033.stdout:3066: writing smithi03385152-26 from 4308992 to 4784128 tid 2 2024-09-21T10:01:11.623 INFO:tasks.rados.rados.0.smithi033.stdout:3066: writing smithi03385152-26 from 4784128 to 5316608 tid 3 2024-09-21T10:01:11.624 INFO:tasks.rados.rados.0.smithi033.stdout:3066: writing smithi03385152-26 from 5316608 to 5513216 tid 4 2024-09-21T10:01:11.624 INFO:tasks.rados.rados.0.smithi033.stdout:3067: read oid 36 snap 280 2024-09-21T10:01:11.624 INFO:tasks.rados.rados.0.smithi033.stdout:3067: expect (ObjNum 91211168 snap 21955 seq_num 91583568) 2024-09-21T10:01:11.624 INFO:tasks.rados.rados.0.smithi033.stdout:3068: snap_create 2024-09-21T10:01:11.624 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1835 (ObjNum 129 snap 0 seq_num 1526604013) dirty exists 2024-09-21T10:01:11.625 INFO:tasks.rados.rados.0.smithi033.stdout:3067: expect (ObjNum 91211168 snap 21955 seq_num 91583568) 2024-09-21T10:01:11.629 INFO:tasks.rados.rados.0.smithi033.stdout:3066: finishing write tid 1 to smithi03385152-26 2024-09-21T10:01:11.629 INFO:tasks.rados.rados.0.smithi033.stdout:3066: finishing write tid 2 to smithi03385152-26 2024-09-21T10:01:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:3066: finishing write tid 3 to smithi03385152-26 2024-09-21T10:01:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:3066: finishing write tid 4 to smithi03385152-26 2024-09-21T10:01:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:3066: finishing write tid 5 to smithi03385152-26 2024-09-21T10:01:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:3066: finishing write tid 6 to smithi03385152-26 2024-09-21T10:01:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1626 (ObjNum 963 snap 282 seq_num 963) dirty exists 2024-09-21T10:01:11.630 INFO:tasks.rados.rados.0.smithi033.stdout:3066: left oid 26 (ObjNum 963 snap 282 seq_num 963) 2024-09-21T10:01:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:11 smithi033 ceph-mon[98546]: pgmap v591: 121 pgs: 121 active+clean; 110 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-09-21T10:01:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:11 smithi033 ceph-mon[98546]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T10:01:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:11 smithi033 ceph-mon[96568]: pgmap v591: 121 pgs: 121 active+clean; 110 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-09-21T10:01:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:11 smithi033 ceph-mon[96568]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T10:01:11.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:01:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:01:11] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-09-21T10:01:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:11 smithi105 ceph-mon[86649]: pgmap v591: 121 pgs: 121 active+clean; 110 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-09-21T10:01:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:11 smithi105 ceph-mon[86649]: osdmap e641: 8 total, 8 up, 8 in 2024-09-21T10:01:12.600 INFO:tasks.rados.rados.0.smithi033.stdout:3065: done (3 left) 2024-09-21T10:01:12.600 INFO:tasks.rados.rados.0.smithi033.stdout:3066: done (2 left) 2024-09-21T10:01:12.600 INFO:tasks.rados.rados.0.smithi033.stdout:3067: done (1 left) 2024-09-21T10:01:12.600 INFO:tasks.rados.rados.0.smithi033.stdout:3068: done (0 left) 2024-09-21T10:01:12.600 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 283 2024-09-21T10:01:12.600 INFO:tasks.rados.rados.0.smithi033.stdout:3069: seq_num 964 ranges {0=548864,548864=761856,1310720=442368} 2024-09-21T10:01:12.603 INFO:tasks.rados.rados.0.smithi033.stdout:3069: writing smithi03385152-20 from 0 to 548864 tid 1 2024-09-21T10:01:12.607 INFO:tasks.rados.rados.0.smithi033.stdout:3069: writing smithi03385152-20 from 548864 to 1310720 tid 2 2024-09-21T10:01:12.610 INFO:tasks.rados.rados.0.smithi033.stdout:3069: writing smithi03385152-20 from 1310720 to 1753088 tid 3 2024-09-21T10:01:12.610 INFO:tasks.rados.rados.0.smithi033.stdout:3070: copy_from oid 12 from oid 47 current snap is 283 2024-09-21T10:01:12.610 INFO:tasks.rados.rados.0.smithi033.stdout:3069: finishing write tid 1 to smithi03385152-20 2024-09-21T10:01:12.610 INFO:tasks.rados.rados.0.smithi033.stdout:3071: read oid 32 snap -1 2024-09-21T10:01:12.610 INFO:tasks.rados.rados.0.smithi033.stdout:3071: expect (ObjNum 943 snap 276 seq_num 943) 2024-09-21T10:01:12.610 INFO:tasks.rados.rados.0.smithi033.stdout:3072: snap_remove snap 282 2024-09-21T10:01:12.898 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:12 smithi033 ceph-mon[98546]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T10:01:12.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:12 smithi033 ceph-mon[96568]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T10:01:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:12 smithi105 ceph-mon[86649]: osdmap e642: 8 total, 8 up, 8 in 2024-09-21T10:01:13.536 INFO:tasks.rados.rados.0.smithi033.stdout:3069: finishing write tid 2 to smithi03385152-20 2024-09-21T10:01:13.536 INFO:tasks.rados.rados.0.smithi033.stdout:3070: finishing copy_from to smithi03385152-12 2024-09-21T10:01:13.536 INFO:tasks.rados.rados.0.smithi033.stdout:3070: got expected ENOENT (src dne) 2024-09-21T10:01:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:3070: finishing copy_from racing read to smithi03385152-12 2024-09-21T10:01:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:3069: finishing write tid 3 to smithi03385152-20 2024-09-21T10:01:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:3069: finishing write tid 4 to smithi03385152-20 2024-09-21T10:01:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:3069: finishing write tid 5 to smithi03385152-20 2024-09-21T10:01:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1620 (ObjNum 964 snap 283 seq_num 964) dirty exists 2024-09-21T10:01:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:3069: left oid 20 (ObjNum 964 snap 283 seq_num 964) 2024-09-21T10:01:13.537 INFO:tasks.rados.rados.0.smithi033.stdout:3071: expect (ObjNum 943 snap 276 seq_num 943) 2024-09-21T10:01:13.576 INFO:tasks.rados.rados.0.smithi033.stdout:3069: done (3 left) 2024-09-21T10:01:13.576 INFO:tasks.rados.rados.0.smithi033.stdout:3070: done (2 left) 2024-09-21T10:01:13.576 INFO:tasks.rados.rados.0.smithi033.stdout:3071: done (1 left) 2024-09-21T10:01:13.577 INFO:tasks.rados.rados.0.smithi033.stdout:3072: done (0 left) 2024-09-21T10:01:13.577 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 283 2024-09-21T10:01:13.577 INFO:tasks.rados.rados.0.smithi033.stdout:3073: seq_num 965 ranges {2441216=565248,3006464=647168,3653632=548864,4202496=8192} 2024-09-21T10:01:13.592 INFO:tasks.rados.rados.0.smithi033.stdout:3073: writing smithi03385152-50 from 2441216 to 3006464 tid 1 2024-09-21T10:01:13.596 INFO:tasks.rados.rados.0.smithi033.stdout:3073: writing smithi03385152-50 from 3006464 to 3653632 tid 2 2024-09-21T10:01:13.600 INFO:tasks.rados.rados.0.smithi033.stdout:3073: writing smithi03385152-50 from 3653632 to 4202496 tid 3 2024-09-21T10:01:13.600 INFO:tasks.rados.rados.0.smithi033.stdout:3073: writing smithi03385152-50 from 4202496 to 4210688 tid 4 2024-09-21T10:01:13.601 INFO:tasks.rados.rados.0.smithi033.stdout:3074: snap_remove snap 280 2024-09-21T10:01:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:13 smithi033 ceph-mon[98546]: pgmap v594: 121 pgs: 121 active+clean; 125 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.7 MiB/s wr, 30 op/s 2024-09-21T10:01:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:13 smithi033 ceph-mon[98546]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T10:01:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:13 smithi033 ceph-mon[98546]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T10:01:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:13 smithi033 ceph-mon[96568]: pgmap v594: 121 pgs: 121 active+clean; 125 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.7 MiB/s wr, 30 op/s 2024-09-21T10:01:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:13 smithi033 ceph-mon[96568]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T10:01:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:13 smithi033 ceph-mon[96568]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T10:01:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:13 smithi105 ceph-mon[86649]: pgmap v594: 121 pgs: 121 active+clean; 125 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.7 MiB/s wr, 30 op/s 2024-09-21T10:01:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:13 smithi105 ceph-mon[86649]: osdmap e643: 8 total, 8 up, 8 in 2024-09-21T10:01:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:13 smithi105 ceph-mon[86649]: osdmap e644: 8 total, 8 up, 8 in 2024-09-21T10:01:14.607 INFO:tasks.rados.rados.0.smithi033.stdout:3074: done (1 left) 2024-09-21T10:01:14.607 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 283 2024-09-21T10:01:14.608 INFO:tasks.rados.rados.0.smithi033.stdout:3075: seq_num 966 ranges {1286144=581632,1867776=581632,2449408=622592} 2024-09-21T10:01:14.618 INFO:tasks.rados.rados.0.smithi033.stdout:3075: writing smithi03385152-18 from 1286144 to 1867776 tid 1 2024-09-21T10:01:14.622 INFO:tasks.rados.rados.0.smithi033.stdout:3075: writing smithi03385152-18 from 1867776 to 2449408 tid 2 2024-09-21T10:01:14.626 INFO:tasks.rados.rados.0.smithi033.stdout:3075: writing smithi03385152-18 from 2449408 to 3072000 tid 3 2024-09-21T10:01:14.626 INFO:tasks.rados.rados.0.smithi033.stdout:3076: snap_create 2024-09-21T10:01:14.626 INFO:tasks.rados.rados.0.smithi033.stdout:3073: finishing write tid 1 to smithi03385152-50 2024-09-21T10:01:14.626 INFO:tasks.rados.rados.0.smithi033.stdout:3073: finishing write tid 2 to smithi03385152-50 2024-09-21T10:01:14.626 INFO:tasks.rados.rados.0.smithi033.stdout:3073: finishing write tid 3 to smithi03385152-50 2024-09-21T10:01:14.626 INFO:tasks.rados.rados.0.smithi033.stdout:3073: finishing write tid 4 to smithi03385152-50 2024-09-21T10:01:14.626 INFO:tasks.rados.rados.0.smithi033.stdout:3073: finishing write tid 5 to smithi03385152-50 2024-09-21T10:01:14.626 INFO:tasks.rados.rados.0.smithi033.stdout:3073: finishing write tid 6 to smithi03385152-50 2024-09-21T10:01:14.627 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 1800 (ObjNum 965 snap 283 seq_num 965) dirty exists 2024-09-21T10:01:14.627 INFO:tasks.rados.rados.0.smithi033.stdout:3073: left oid 50 (ObjNum 965 snap 283 seq_num 965) 2024-09-21T10:01:14.627 INFO:tasks.rados.rados.0.smithi033.stdout:3075: finishing write tid 1 to smithi03385152-18 2024-09-21T10:01:14.627 INFO:tasks.rados.rados.0.smithi033.stdout:3075: finishing write tid 2 to smithi03385152-18 2024-09-21T10:01:14.628 INFO:tasks.rados.rados.0.smithi033.stdout:3075: finishing write tid 3 to smithi03385152-18 2024-09-21T10:01:14.628 INFO:tasks.rados.rados.0.smithi033.stdout:3075: finishing write tid 4 to smithi03385152-18 2024-09-21T10:01:14.632 INFO:tasks.rados.rados.0.smithi033.stdout:3075: finishing write tid 5 to smithi03385152-18 2024-09-21T10:01:14.632 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1024 (ObjNum 966 snap 283 seq_num 966) dirty exists 2024-09-21T10:01:14.632 INFO:tasks.rados.rados.0.smithi033.stdout:3075: left oid 18 (ObjNum 966 snap 283 seq_num 966) 2024-09-21T10:01:15.608 INFO:tasks.rados.rados.0.smithi033.stdout:3073: done (2 left) 2024-09-21T10:01:15.608 INFO:tasks.rados.rados.0.smithi033.stdout:3075: done (1 left) 2024-09-21T10:01:15.609 INFO:tasks.rados.rados.0.smithi033.stdout:3076: done (0 left) 2024-09-21T10:01:15.609 INFO:tasks.rados.rados.0.smithi033.stdout:3077: read oid 30 snap -1 2024-09-21T10:01:15.609 INFO:tasks.rados.rados.0.smithi033.stdout:3077: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:15.609 INFO:tasks.rados.rados.0.smithi033.stdout:3078: delete oid 49 current snap is 284 2024-09-21T10:01:15.614 INFO:tasks.rados.rados.0.smithi033.stdout:3078: done (1 left) 2024-09-21T10:01:15.614 INFO:tasks.rados.rados.0.smithi033.stdout:3079: rollback oid 33 current snap is 284 2024-09-21T10:01:15.614 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 279 2024-09-21T10:01:15.614 INFO:tasks.rados.rados.0.smithi033.stdout:3080: copy_from oid 25 from oid 40 current snap is 284 2024-09-21T10:01:15.615 INFO:tasks.rados.rados.0.smithi033.stdout:3081: read oid 8 snap -1 2024-09-21T10:01:15.615 INFO:tasks.rados.rados.0.smithi033.stdout:3081: expect (ObjNum 920 snap 268 seq_num 920) 2024-09-21T10:01:15.615 INFO:tasks.rados.rados.0.smithi033.stdout:3082: delete oid 36 current snap is 284 2024-09-21T10:01:15.615 INFO:tasks.rados.rados.0.smithi033.stdout:3077: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:15.618 INFO:tasks.rados.rados.0.smithi033.stdout:3080: finishing copy_from to smithi03385152-25 2024-09-21T10:01:15.619 INFO:tasks.rados.rados.0.smithi033.stdout:3080: got expected ENOENT (src dne) 2024-09-21T10:01:15.619 INFO:tasks.rados.rados.0.smithi033.stdout:3080: finishing copy_from racing read to smithi03385152-25 2024-09-21T10:01:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:3077: done (4 left) 2024-09-21T10:01:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:3080: done (3 left) 2024-09-21T10:01:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:3082: done (2 left) 2024-09-21T10:01:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:3083: rollback oid 30 current snap is 284 2024-09-21T10:01:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 30 to 283 2024-09-21T10:01:15.620 INFO:tasks.rados.rados.0.smithi033.stdout:3079: finishing rollback tid 0 to smithi03385152-33 2024-09-21T10:01:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:3084: read oid 26 snap -1 2024-09-21T10:01:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:3084: expect (ObjNum 963 snap 282 seq_num 963) 2024-09-21T10:01:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:3085: read oid 12 snap 272 2024-09-21T10:01:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:3085: expect (ObjNum 893 snap 257 seq_num 893) 2024-09-21T10:01:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:3086: rmattr oid 38 current snap is 284 2024-09-21T10:01:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:3087: snap_create 2024-09-21T10:01:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:3079: finishing rollback tid 1 to smithi03385152-33 2024-09-21T10:01:15.621 INFO:tasks.rados.rados.0.smithi033.stdout:3079: finishing rollback tid 2 to smithi03385152-33 2024-09-21T10:01:15.622 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1619 (ObjNum 919 snap 268 seq_num 919) dirty exists 2024-09-21T10:01:15.624 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 1803 (ObjNum 1163893984 snap 32666 seq_num 304) dirty exists 2024-09-21T10:01:15.627 INFO:tasks.rados.rados.0.smithi033.stdout:3081: expect (ObjNum 920 snap 268 seq_num 920) 2024-09-21T10:01:15.691 INFO:tasks.rados.rados.0.smithi033.stdout:3083: finishing rollback tid 0 to smithi03385152-30 2024-09-21T10:01:15.691 INFO:tasks.rados.rados.0.smithi033.stdout:3083: finishing rollback tid 1 to smithi03385152-30 2024-09-21T10:01:15.691 INFO:tasks.rados.rados.0.smithi033.stdout:3083: finishing rollback tid 2 to smithi03385152-30 2024-09-21T10:01:15.691 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2278 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:01:15.691 INFO:tasks.rados.rados.0.smithi033.stdout:3084: expect (ObjNum 963 snap 282 seq_num 963) 2024-09-21T10:01:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:15 smithi033 ceph-mon[98546]: pgmap v597: 121 pgs: 121 active+clean; 122 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T10:01:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:15 smithi033 ceph-mon[98546]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T10:01:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:15 smithi033 ceph-mon[96568]: pgmap v597: 121 pgs: 121 active+clean; 122 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T10:01:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:15 smithi033 ceph-mon[96568]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T10:01:15.996 INFO:tasks.rados.rados.0.smithi033.stdout:3085: expect (ObjNum 893 snap 257 seq_num 893) 2024-09-21T10:01:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:15 smithi105 ceph-mon[86649]: pgmap v597: 121 pgs: 121 active+clean; 122 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.7 MiB/s wr, 32 op/s 2024-09-21T10:01:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:15 smithi105 ceph-mon[86649]: osdmap e645: 8 total, 8 up, 8 in 2024-09-21T10:01:16.621 INFO:tasks.rados.rados.0.smithi033.stdout:3079: done (6 left) 2024-09-21T10:01:16.621 INFO:tasks.rados.rados.0.smithi033.stdout:3081: done (5 left) 2024-09-21T10:01:16.621 INFO:tasks.rados.rados.0.smithi033.stdout:3083: done (4 left) 2024-09-21T10:01:16.621 INFO:tasks.rados.rados.0.smithi033.stdout:3084: done (3 left) 2024-09-21T10:01:16.621 INFO:tasks.rados.rados.0.smithi033.stdout:3085: done (2 left) 2024-09-21T10:01:16.621 INFO:tasks.rados.rados.0.smithi033.stdout:3086: done (1 left) 2024-09-21T10:01:16.621 INFO:tasks.rados.rados.0.smithi033.stdout:3087: done (0 left) 2024-09-21T10:01:16.621 INFO:tasks.rados.rados.0.smithi033.stdout:3088: snap_remove snap 281 2024-09-21T10:01:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:16 smithi033 ceph-mon[98546]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T10:01:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:16 smithi033 ceph-mon[96568]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T10:01:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:16 smithi105 ceph-mon[86649]: osdmap e646: 8 total, 8 up, 8 in 2024-09-21T10:01:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:17.625 INFO:tasks.rados.rados.0.smithi033.stdout:3088: done (0 left) 2024-09-21T10:01:17.625 INFO:tasks.rados.rados.0.smithi033.stdout:3089: setattr oid 32 current snap is 285 2024-09-21T10:01:17.628 INFO:tasks.rados.rados.0.smithi033.stdout:3090: copy_from oid 38 from oid 50 current snap is 285 2024-09-21T10:01:17.629 INFO:tasks.rados.rados.0.smithi033.stdout:3091: read oid 1 snap 275 2024-09-21T10:01:17.629 INFO:tasks.rados.rados.0.smithi033.stdout:3091: expect (ObjNum 723 snap 205 seq_num 723) 2024-09-21T10:01:17.629 INFO:tasks.rados.rados.0.smithi033.stdout:3092: read oid 40 snap 272 2024-09-21T10:01:17.629 INFO:tasks.rados.rados.0.smithi033.stdout:3092: expect deleted 2024-09-21T10:01:17.629 INFO:tasks.rados.rados.0.smithi033.stdout:3093: rollback oid 6 current snap is 285 2024-09-21T10:01:17.629 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 284 2024-09-21T10:01:17.629 INFO:tasks.rados.rados.0.smithi033.stdout:3094: snap_remove snap 279 2024-09-21T10:01:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:17 smithi033 ceph-mon[98546]: pgmap v600: 121 pgs: 121 active+clean; 130 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-21T10:01:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:17 smithi033 ceph-mon[98546]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T10:01:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:17 smithi033 ceph-mon[96568]: pgmap v600: 121 pgs: 121 active+clean; 130 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-21T10:01:17.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:17 smithi033 ceph-mon[96568]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T10:01:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:17 smithi105 ceph-mon[86649]: pgmap v600: 121 pgs: 121 active+clean; 130 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.9 MiB/s wr, 31 op/s 2024-09-21T10:01:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:17 smithi105 ceph-mon[86649]: osdmap e647: 8 total, 8 up, 8 in 2024-09-21T10:01:18.539 INFO:tasks.rados.rados.0.smithi033.stdout:3094: done (5 left) 2024-09-21T10:01:18.539 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 285 2024-09-21T10:01:18.539 INFO:tasks.rados.rados.0.smithi033.stdout:3091: expect (ObjNum 723 snap 205 seq_num 723) 2024-09-21T10:01:18.547 INFO:tasks.rados.rados.0.smithi033.stdout:3093: finishing rollback tid 1 to smithi03385152-6 2024-09-21T10:01:18.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 0 (ObjNum 129 snap 0 seq_num 91029776) dirty dne 2024-09-21T10:01:18.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1587 (ObjNum 943 snap 276 seq_num 943) dirty exists 2024-09-21T10:01:18.547 INFO:tasks.rados.rados.0.smithi033.stdout:3090: finishing copy_from racing read to smithi03385152-38 2024-09-21T10:01:18.547 INFO:tasks.rados.rados.0.smithi033.stdout:3090: finishing copy_from to smithi03385152-38 2024-09-21T10:01:18.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 1800 (ObjNum 965 snap 283 seq_num 965) dirty exists 2024-09-21T10:01:18.547 INFO:tasks.rados.rados.0.smithi033.stdout:3095: seq_num 969 ranges {1335296=638976,1974272=630784,2605056=532480,3137536=40960} 2024-09-21T10:01:18.557 INFO:tasks.rados.rados.0.smithi033.stdout:3095: writing smithi03385152-44 from 1335296 to 1974272 tid 1 2024-09-21T10:01:18.561 INFO:tasks.rados.rados.0.smithi033.stdout:3095: writing smithi03385152-44 from 1974272 to 2605056 tid 2 2024-09-21T10:01:18.564 INFO:tasks.rados.rados.0.smithi033.stdout:3095: writing smithi03385152-44 from 2605056 to 3137536 tid 3 2024-09-21T10:01:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:3095: writing smithi03385152-44 from 3137536 to 3178496 tid 4 2024-09-21T10:01:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:3089: done (5 left) 2024-09-21T10:01:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:3090: done (4 left) 2024-09-21T10:01:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:3091: done (3 left) 2024-09-21T10:01:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:3092: done (2 left) 2024-09-21T10:01:18.565 INFO:tasks.rados.rados.0.smithi033.stdout:3093: done (1 left) 2024-09-21T10:01:18.566 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 285 2024-09-21T10:01:18.566 INFO:tasks.rados.rados.0.smithi033.stdout:3095: finishing write tid 1 to smithi03385152-44 2024-09-21T10:01:18.566 INFO:tasks.rados.rados.0.smithi033.stdout:3095: finishing write tid 2 to smithi03385152-44 2024-09-21T10:01:18.566 INFO:tasks.rados.rados.0.smithi033.stdout:3096: seq_num 970 ranges {0=655360,655360=516096,1171456=688128} 2024-09-21T10:01:18.568 INFO:tasks.rados.rados.0.smithi033.stdout:3096: writing smithi03385152-9 from 0 to 655360 tid 1 2024-09-21T10:01:18.572 INFO:tasks.rados.rados.0.smithi033.stdout:3096: writing smithi03385152-9 from 655360 to 1171456 tid 2 2024-09-21T10:01:18.576 INFO:tasks.rados.rados.0.smithi033.stdout:3096: writing smithi03385152-9 from 1171456 to 1859584 tid 3 2024-09-21T10:01:18.576 INFO:tasks.rados.rados.0.smithi033.stdout:3097: setattr oid 42 current snap is 285 2024-09-21T10:01:18.576 INFO:tasks.rados.rados.0.smithi033.stdout:3095: finishing write tid 3 to smithi03385152-44 2024-09-21T10:01:18.576 INFO:tasks.rados.rados.0.smithi033.stdout:3095: finishing write tid 4 to smithi03385152-44 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:3095: finishing write tid 5 to smithi03385152-44 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:3095: finishing write tid 6 to smithi03385152-44 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 1977 (ObjNum 969 snap 285 seq_num 969) dirty exists 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:3095: left oid 44 (ObjNum 969 snap 285 seq_num 969) 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:3096: finishing write tid 1 to smithi03385152-9 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:3096: finishing write tid 2 to smithi03385152-9 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:3095: done (2 left) 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:3098: rmattr oid 8 current snap is 285 2024-09-21T10:01:18.577 INFO:tasks.rados.rados.0.smithi033.stdout:3099: copy_from oid 29 from oid 20 current snap is 285 2024-09-21T10:01:18.578 INFO:tasks.rados.rados.0.smithi033.stdout:3100: snap_remove snap 283 2024-09-21T10:01:18.999 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:18 smithi033 ceph-mon[98546]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T10:01:19.000 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:18 smithi033 ceph-mon[98546]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T10:01:19.001 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:18 smithi033 ceph-mon[96568]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T10:01:19.002 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:18 smithi033 ceph-mon[96568]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T10:01:19.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:18 smithi105 ceph-mon[86649]: osdmap e648: 8 total, 8 up, 8 in 2024-09-21T10:01:19.032 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:18 smithi105 ceph-mon[86649]: osdmap e649: 8 total, 8 up, 8 in 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:3100: done (4 left) 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:3101: rollback oid 46 current snap is 285 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 46 to 272 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 1503 (ObjNum 920 snap 268 seq_num 920) dirty exists 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:3096: finishing write tid 3 to smithi03385152-9 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:3096: finishing write tid 4 to smithi03385152-9 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 1808 (ObjNum 950 snap 279 seq_num 950) dirty exists 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:3096: finishing write tid 5 to smithi03385152-9 2024-09-21T10:01:19.542 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 1807 (ObjNum 970 snap 285 seq_num 970) dirty exists 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3096: left oid 9 (ObjNum 970 snap 285 seq_num 970) 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3099: finishing copy_from racing read to smithi03385152-29 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3099: finishing copy_from to smithi03385152-29 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1620 (ObjNum 964 snap 283 seq_num 964) dirty exists 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3096: done (4 left) 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3097: done (3 left) 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3098: done (2 left) 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3099: done (1 left) 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3102: rollback oid 26 current snap is 285 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 26 to 284 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3103: rollback oid 28 current snap is 285 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 272 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 285 2024-09-21T10:01:19.543 INFO:tasks.rados.rados.0.smithi033.stdout:3104: seq_num 971 ranges {933888=671744,1605632=802816,2408448=401408} 2024-09-21T10:01:19.551 INFO:tasks.rados.rados.0.smithi033.stdout:3104: writing smithi03385152-37 from 933888 to 1605632 tid 1 2024-09-21T10:01:19.557 INFO:tasks.rados.rados.0.smithi033.stdout:3104: writing smithi03385152-37 from 1605632 to 2408448 tid 2 2024-09-21T10:01:19.561 INFO:tasks.rados.rados.0.smithi033.stdout:3104: writing smithi03385152-37 from 2408448 to 2809856 tid 3 2024-09-21T10:01:19.561 INFO:tasks.rados.rados.0.smithi033.stdout:3105: read oid 49 snap -1 2024-09-21T10:01:19.561 INFO:tasks.rados.rados.0.smithi033.stdout:3105: expect deleted 2024-09-21T10:01:19.561 INFO:tasks.rados.rados.0.smithi033.stdout:3106: snap_remove snap 275 2024-09-21T10:01:19.561 INFO:tasks.rados.rados.0.smithi033.stdout:3101: finishing rollback tid 1 to smithi03385152-46 2024-09-21T10:01:19.561 INFO:tasks.rados.rados.0.smithi033.stdout:3101: finishing rollback tid 2 to smithi03385152-46 2024-09-21T10:01:19.562 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1845 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:01:19.562 INFO:tasks.rados.rados.0.smithi033.stdout:3102: finishing rollback tid 0 to smithi03385152-26 2024-09-21T10:01:19.562 INFO:tasks.rados.rados.0.smithi033.stdout:3103: finishing rollback tid 0 to smithi03385152-28 2024-09-21T10:01:19.562 INFO:tasks.rados.rados.0.smithi033.stdout:3103: finishing rollback tid 1 to smithi03385152-28 2024-09-21T10:01:19.562 INFO:tasks.rados.rados.0.smithi033.stdout:3103: finishing rollback tid 2 to smithi03385152-28 2024-09-21T10:01:19.562 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 915 (ObjNum 914 snap 265 seq_num 914) dirty exists 2024-09-21T10:01:19.563 INFO:tasks.rados.rados.0.smithi033.stdout:3102: finishing rollback tid 1 to smithi03385152-26 2024-09-21T10:01:19.563 INFO:tasks.rados.rados.0.smithi033.stdout:3102: finishing rollback tid 2 to smithi03385152-26 2024-09-21T10:01:19.563 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1630 (ObjNum 963 snap 282 seq_num 963) dirty exists 2024-09-21T10:01:19.563 INFO:tasks.rados.rados.0.smithi033.stdout:3104: finishing write tid 1 to smithi03385152-37 2024-09-21T10:01:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:19 smithi033 ceph-mon[96568]: pgmap v603: 121 pgs: 121 active+clean; 131 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.3 MiB/s wr, 15 op/s 2024-09-21T10:01:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:19 smithi033 ceph-mon[96568]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T10:01:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:19 smithi033 ceph-mon[98546]: pgmap v603: 121 pgs: 121 active+clean; 131 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.3 MiB/s wr, 15 op/s 2024-09-21T10:01:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:19 smithi033 ceph-mon[98546]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T10:01:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:19 smithi105 ceph-mon[86649]: pgmap v603: 121 pgs: 121 active+clean; 131 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.3 MiB/s wr, 15 op/s 2024-09-21T10:01:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:19 smithi105 ceph-mon[86649]: osdmap e650: 8 total, 8 up, 8 in 2024-09-21T10:01:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:01:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:01:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:20.543 INFO:tasks.rados.rados.0.smithi033.stdout:3101: done (5 left) 2024-09-21T10:01:20.543 INFO:tasks.rados.rados.0.smithi033.stdout:3102: done (4 left) 2024-09-21T10:01:20.543 INFO:tasks.rados.rados.0.smithi033.stdout:3103: done (3 left) 2024-09-21T10:01:20.543 INFO:tasks.rados.rados.0.smithi033.stdout:3106: done (2 left) 2024-09-21T10:01:20.543 INFO:tasks.rados.rados.0.smithi033.stdout:3107: read oid 15 snap -1 2024-09-21T10:01:20.544 INFO:tasks.rados.rados.0.smithi033.stdout:3107: expect (ObjNum 962 snap 281 seq_num 962) 2024-09-21T10:01:20.544 INFO:tasks.rados.rados.0.smithi033.stdout:3108: delete oid 11 current snap is 285 2024-09-21T10:01:20.544 INFO:tasks.rados.rados.0.smithi033.stdout:3104: finishing write tid 2 to smithi03385152-37 2024-09-21T10:01:20.544 INFO:tasks.rados.rados.0.smithi033.stdout:3104: finishing write tid 3 to smithi03385152-37 2024-09-21T10:01:20.544 INFO:tasks.rados.rados.0.smithi033.stdout:3104: finishing write tid 4 to smithi03385152-37 2024-09-21T10:01:20.544 INFO:tasks.rados.rados.0.smithi033.stdout:3104: finishing write tid 5 to smithi03385152-37 2024-09-21T10:01:20.544 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1523 (ObjNum 971 snap 285 seq_num 971) dirty exists 2024-09-21T10:01:20.544 INFO:tasks.rados.rados.0.smithi033.stdout:3104: left oid 37 (ObjNum 971 snap 285 seq_num 971) 2024-09-21T10:01:20.552 INFO:tasks.rados.rados.0.smithi033.stdout:3104: done (3 left) 2024-09-21T10:01:20.552 INFO:tasks.rados.rados.0.smithi033.stdout:3105: done (2 left) 2024-09-21T10:01:20.552 INFO:tasks.rados.rados.0.smithi033.stdout:3108: done (1 left) 2024-09-21T10:01:20.552 INFO:tasks.rados.rados.0.smithi033.stdout:3109: snap_remove snap 272 2024-09-21T10:01:21.547 INFO:tasks.rados.rados.0.smithi033.stdout:3107: expect (ObjNum 962 snap 281 seq_num 962) 2024-09-21T10:01:21.770 INFO:tasks.rados.rados.0.smithi033.stdout:3107: done (1 left) 2024-09-21T10:01:21.770 INFO:tasks.rados.rados.0.smithi033.stdout:3109: done (0 left) 2024-09-21T10:01:21.770 INFO:tasks.rados.rados.0.smithi033.stdout:3110: rollback oid 37 current snap is 285 2024-09-21T10:01:21.770 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 284 2024-09-21T10:01:21.770 INFO:tasks.rados.rados.0.smithi033.stdout:3111: copy_from oid 8 from oid 36 current snap is 285 2024-09-21T10:01:21.770 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 285 2024-09-21T10:01:21.770 INFO:tasks.rados.rados.0.smithi033.stdout:3112: seq_num 973 ranges {0=704512,704512=573440,1277952=507904,1785856=122880} 2024-09-21T10:01:21.773 INFO:tasks.rados.rados.0.smithi033.stdout:3112: writing smithi03385152-31 from 0 to 704512 tid 1 2024-09-21T10:01:21.777 INFO:tasks.rados.rados.0.smithi033.stdout:3112: writing smithi03385152-31 from 704512 to 1277952 tid 2 2024-09-21T10:01:21.780 INFO:tasks.rados.rados.0.smithi033.stdout:3112: writing smithi03385152-31 from 1277952 to 1785856 tid 3 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3112: writing smithi03385152-31 from 1785856 to 1908736 tid 4 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3113: rmattr oid 17 current snap is 285 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3113: done (3 left) 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3114: delete oid 12 current snap is 285 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3111: finishing copy_from to smithi03385152-8 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3111: got expected ENOENT (src dne) 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3111: finishing copy_from racing read to smithi03385152-8 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3110: finishing rollback tid 0 to smithi03385152-37 2024-09-21T10:01:21.781 INFO:tasks.rados.rados.0.smithi033.stdout:3110: finishing rollback tid 1 to smithi03385152-37 2024-09-21T10:01:21.782 INFO:tasks.rados.rados.0.smithi033.stdout:3110: finishing rollback tid 2 to smithi03385152-37 2024-09-21T10:01:21.782 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 1527 (ObjNum 915 snap 265 seq_num 915) dirty exists 2024-09-21T10:01:21.782 INFO:tasks.rados.rados.0.smithi033.stdout:3112: finishing write tid 1 to smithi03385152-31 2024-09-21T10:01:21.782 INFO:tasks.rados.rados.0.smithi033.stdout:3112: finishing write tid 2 to smithi03385152-31 2024-09-21T10:01:21.783 INFO:tasks.rados.rados.0.smithi033.stdout:3110: done (3 left) 2024-09-21T10:01:21.783 INFO:tasks.rados.rados.0.smithi033.stdout:3111: done (2 left) 2024-09-21T10:01:21.783 INFO:tasks.rados.rados.0.smithi033.stdout:3114: done (1 left) 2024-09-21T10:01:21.784 INFO:tasks.rados.rados.0.smithi033.stdout:3115: snap_remove snap 284 2024-09-21T10:01:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:21 smithi033 ceph-mon[98546]: pgmap v606: 121 pgs: 121 active+clean; 133 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-21T10:01:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:21 smithi033 ceph-mon[98546]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T10:01:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:21 smithi033 ceph-mon[96568]: pgmap v606: 121 pgs: 121 active+clean; 133 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-21T10:01:21.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:21 smithi033 ceph-mon[96568]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T10:01:21.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:01:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:01:21] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-09-21T10:01:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:21 smithi105 ceph-mon[86649]: pgmap v606: 121 pgs: 121 active+clean; 133 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-21T10:01:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:21 smithi105 ceph-mon[86649]: osdmap e651: 8 total, 8 up, 8 in 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:3115: done (1 left) 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 285 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:3112: finishing write tid 3 to smithi03385152-31 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:3112: finishing write tid 4 to smithi03385152-31 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:3112: finishing write tid 5 to smithi03385152-31 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:3112: finishing write tid 6 to smithi03385152-31 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2324 (ObjNum 973 snap 285 seq_num 973) dirty exists 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:3112: left oid 31 (ObjNum 973 snap 285 seq_num 973) 2024-09-21T10:01:22.557 INFO:tasks.rados.rados.0.smithi033.stdout:3116: seq_num 975 ranges {1753088=737280,2490368=786432,3276800=417792} 2024-09-21T10:01:22.570 INFO:tasks.rados.rados.0.smithi033.stdout:3116: writing smithi03385152-29 from 1753088 to 2490368 tid 1 2024-09-21T10:01:22.578 INFO:tasks.rados.rados.0.smithi033.stdout:3116: writing smithi03385152-29 from 2490368 to 3276800 tid 2 2024-09-21T10:01:22.578 INFO:tasks.rados.rados.0.smithi033.stdout:3116: writing smithi03385152-29 from 3276800 to 3694592 tid 3 2024-09-21T10:01:22.578 INFO:tasks.rados.rados.0.smithi033.stdout:3112: done (1 left) 2024-09-21T10:01:22.578 INFO:tasks.rados.rados.0.smithi033.stdout:3117: rmattr oid 43 current snap is 285 2024-09-21T10:01:22.578 INFO:tasks.rados.rados.0.smithi033.stdout:3116: finishing write tid 1 to smithi03385152-29 2024-09-21T10:01:22.578 INFO:tasks.rados.rados.0.smithi033.stdout:3118: rollback oid 30 current snap is 285 2024-09-21T10:01:22.579 INFO:tasks.rados.rados.0.smithi033.stdout:3118: done (2 left) 2024-09-21T10:01:22.579 INFO:tasks.rados.rados.0.smithi033.stdout:3119: delete oid 2 current snap is 285 2024-09-21T10:01:22.580 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1528 (ObjNum 949 snap 279 seq_num 949) dirty exists 2024-09-21T10:01:22.580 INFO:tasks.rados.rados.0.smithi033.stdout:3117: done (2 left) 2024-09-21T10:01:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3119: done (1 left) 2024-09-21T10:01:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3120: rollback oid 49 current snap is 285 2024-09-21T10:01:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3120: done (1 left) 2024-09-21T10:01:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3121: copy_from oid 3 from oid 20 current snap is 285 2024-09-21T10:01:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3122: delete oid 10 current snap is 285 2024-09-21T10:01:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3116: finishing write tid 2 to smithi03385152-29 2024-09-21T10:01:22.582 INFO:tasks.rados.rados.0.smithi033.stdout:3116: finishing write tid 3 to smithi03385152-29 2024-09-21T10:01:22.582 INFO:tasks.rados.rados.0.smithi033.stdout:3116: finishing write tid 4 to smithi03385152-29 2024-09-21T10:01:22.584 INFO:tasks.rados.rados.0.smithi033.stdout:3116: finishing write tid 5 to smithi03385152-29 2024-09-21T10:01:22.584 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2332 (ObjNum 975 snap 285 seq_num 975) dirty exists 2024-09-21T10:01:22.584 INFO:tasks.rados.rados.0.smithi033.stdout:3116: left oid 29 (ObjNum 975 snap 285 seq_num 975) 2024-09-21T10:01:22.586 INFO:tasks.rados.rados.0.smithi033.stdout:3116: done (2 left) 2024-09-21T10:01:22.586 INFO:tasks.rados.rados.0.smithi033.stdout:3122: done (1 left) 2024-09-21T10:01:22.586 INFO:tasks.rados.rados.0.smithi033.stdout:3123: copy_from oid 31 from oid 48 current snap is 285 2024-09-21T10:01:22.586 INFO:tasks.rados.rados.0.smithi033.stdout:3124: delete oid 44 current snap is 285 2024-09-21T10:01:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:3124: done (2 left) 2024-09-21T10:01:22.589 INFO:tasks.rados.rados.0.smithi033.stdout:3125: read oid 35 snap -1 2024-09-21T10:01:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:3125: expect deleted 2024-09-21T10:01:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 285 2024-09-21T10:01:22.590 INFO:tasks.rados.rados.0.smithi033.stdout:3126: seq_num 979 ranges {5513216=802816,6316032=729088,7045120=458752,7503872=16384} 2024-09-21T10:01:22.625 INFO:tasks.rados.rados.0.smithi033.stdout:3126: writing smithi03385152-26 from 5513216 to 6316032 tid 1 2024-09-21T10:01:22.629 INFO:tasks.rados.rados.0.smithi033.stdout:3126: writing smithi03385152-26 from 6316032 to 7045120 tid 2 2024-09-21T10:01:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:3126: writing smithi03385152-26 from 7045120 to 7503872 tid 3 2024-09-21T10:01:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:3126: writing smithi03385152-26 from 7503872 to 7520256 tid 4 2024-09-21T10:01:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 285 2024-09-21T10:01:22.632 INFO:tasks.rados.rados.0.smithi033.stdout:3127: seq_num 980 ranges {0=417792,417792=614400,1032192=663552,1695744=327680} 2024-09-21T10:01:22.634 INFO:tasks.rados.rados.0.smithi033.stdout:3127: writing smithi03385152-6 from 0 to 417792 tid 1 2024-09-21T10:01:22.638 INFO:tasks.rados.rados.0.smithi033.stdout:3127: writing smithi03385152-6 from 417792 to 1032192 tid 2 2024-09-21T10:01:22.641 INFO:tasks.rados.rados.0.smithi033.stdout:3127: writing smithi03385152-6 from 1032192 to 1695744 tid 3 2024-09-21T10:01:22.644 INFO:tasks.rados.rados.0.smithi033.stdout:3127: writing smithi03385152-6 from 1695744 to 2023424 tid 4 2024-09-21T10:01:22.644 INFO:tasks.rados.rados.0.smithi033.stdout:3128: copy_from oid 32 from oid 4 current snap is 285 2024-09-21T10:01:22.644 INFO:tasks.rados.rados.0.smithi033.stdout:3121: finishing copy_from racing read to smithi03385152-3 2024-09-21T10:01:22.644 INFO:tasks.rados.rados.0.smithi033.stdout:3121: finishing copy_from to smithi03385152-3 2024-09-21T10:01:22.644 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 1620 (ObjNum 964 snap 283 seq_num 964) dirty exists 2024-09-21T10:01:22.644 INFO:tasks.rados.rados.0.smithi033.stdout:3123: finishing copy_from racing read to smithi03385152-31 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3121: done (5 left) 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3125: done (4 left) 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3129: snap_create 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3123: finishing copy_from to smithi03385152-31 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 1834 (ObjNum 909 snap 263 seq_num 909) dirty exists 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3126: finishing write tid 1 to smithi03385152-26 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3126: finishing write tid 2 to smithi03385152-26 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3126: finishing write tid 3 to smithi03385152-26 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3126: finishing write tid 4 to smithi03385152-26 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3126: finishing write tid 5 to smithi03385152-26 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3127: finishing write tid 1 to smithi03385152-6 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3126: finishing write tid 6 to smithi03385152-26 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1636 (ObjNum 979 snap 285 seq_num 979) dirty exists 2024-09-21T10:01:22.645 INFO:tasks.rados.rados.0.smithi033.stdout:3126: left oid 26 (ObjNum 979 snap 285 seq_num 979) 2024-09-21T10:01:22.646 INFO:tasks.rados.rados.0.smithi033.stdout:3127: finishing write tid 2 to smithi03385152-6 2024-09-21T10:01:22.646 INFO:tasks.rados.rados.0.smithi033.stdout:3128: finishing copy_from to smithi03385152-32 2024-09-21T10:01:22.646 INFO:tasks.rados.rados.0.smithi033.stdout:3128: got expected ENOENT (src dne) 2024-09-21T10:01:22.646 INFO:tasks.rados.rados.0.smithi033.stdout:3128: finishing copy_from racing read to smithi03385152-32 2024-09-21T10:01:22.646 INFO:tasks.rados.rados.0.smithi033.stdout:3127: finishing write tid 3 to smithi03385152-6 2024-09-21T10:01:22.647 INFO:tasks.rados.rados.0.smithi033.stdout:3127: finishing write tid 4 to smithi03385152-6 2024-09-21T10:01:22.647 INFO:tasks.rados.rados.0.smithi033.stdout:3127: finishing write tid 5 to smithi03385152-6 2024-09-21T10:01:22.647 INFO:tasks.rados.rados.0.smithi033.stdout:3127: finishing write tid 6 to smithi03385152-6 2024-09-21T10:01:22.647 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:01:22.647 INFO:tasks.rados.rados.0.smithi033.stdout:3127: left oid 6 (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:01:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:22 smithi033 ceph-mon[98546]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T10:01:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:22 smithi033 ceph-mon[96568]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T10:01:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:22 smithi105 ceph-mon[86649]: osdmap e652: 8 total, 8 up, 8 in 2024-09-21T10:01:23.542 INFO:tasks.rados.rados.0.smithi033.stdout:3123: done (4 left) 2024-09-21T10:01:23.543 INFO:tasks.rados.rados.0.smithi033.stdout:3126: done (3 left) 2024-09-21T10:01:23.543 INFO:tasks.rados.rados.0.smithi033.stdout:3127: done (2 left) 2024-09-21T10:01:23.543 INFO:tasks.rados.rados.0.smithi033.stdout:3128: done (1 left) 2024-09-21T10:01:23.543 INFO:tasks.rados.rados.0.smithi033.stdout:3129: done (0 left) 2024-09-21T10:01:23.543 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 286 2024-09-21T10:01:23.543 INFO:tasks.rados.rados.0.smithi033.stdout:3130: seq_num 981 ranges {2973696=434176,3407872=540672,3948544=466944,4415488=524288,4939776=73728} 2024-09-21T10:01:23.561 INFO:tasks.rados.rados.0.smithi033.stdout:3130: writing smithi03385152-13 from 2973696 to 3407872 tid 1 2024-09-21T10:01:23.564 INFO:tasks.rados.rados.0.smithi033.stdout:3130: writing smithi03385152-13 from 3407872 to 3948544 tid 2 2024-09-21T10:01:23.566 INFO:tasks.rados.rados.0.smithi033.stdout:3130: writing smithi03385152-13 from 3948544 to 4415488 tid 3 2024-09-21T10:01:23.570 INFO:tasks.rados.rados.0.smithi033.stdout:3130: writing smithi03385152-13 from 4415488 to 4939776 tid 4 2024-09-21T10:01:23.571 INFO:tasks.rados.rados.0.smithi033.stdout:3130: writing smithi03385152-13 from 4939776 to 5013504 tid 5 2024-09-21T10:01:23.571 INFO:tasks.rados.rados.0.smithi033.stdout:3130: finishing write tid 1 to smithi03385152-13 2024-09-21T10:01:23.571 INFO:tasks.rados.rados.0.smithi033.stdout:3131: rollback oid 43 current snap is 286 2024-09-21T10:01:23.571 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 43 to 285 2024-09-21T10:01:23.571 INFO:tasks.rados.rados.0.smithi033.stdout:3130: finishing write tid 2 to smithi03385152-13 2024-09-21T10:01:23.571 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 286 2024-09-21T10:01:23.571 INFO:tasks.rados.rados.0.smithi033.stdout:3132: seq_num 982 ranges {57344=450560,507904=425984,933888=679936,1613824=466944,2080768=32768} 2024-09-21T10:01:23.574 INFO:tasks.rados.rados.0.smithi033.stdout:3132: writing smithi03385152-30 from 57344 to 507904 tid 1 2024-09-21T10:01:23.577 INFO:tasks.rados.rados.0.smithi033.stdout:3132: writing smithi03385152-30 from 507904 to 933888 tid 2 2024-09-21T10:01:23.581 INFO:tasks.rados.rados.0.smithi033.stdout:3132: writing smithi03385152-30 from 933888 to 1613824 tid 3 2024-09-21T10:01:23.584 INFO:tasks.rados.rados.0.smithi033.stdout:3132: writing smithi03385152-30 from 1613824 to 2080768 tid 4 2024-09-21T10:01:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3132: writing smithi03385152-30 from 2080768 to 2113536 tid 5 2024-09-21T10:01:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3133: snap_create 2024-09-21T10:01:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3130: finishing write tid 3 to smithi03385152-13 2024-09-21T10:01:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3131: finishing rollback tid 0 to smithi03385152-43 2024-09-21T10:01:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3131: finishing rollback tid 1 to smithi03385152-43 2024-09-21T10:01:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3131: finishing rollback tid 2 to smithi03385152-43 2024-09-21T10:01:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1531 (ObjNum 949 snap 279 seq_num 949) dirty exists 2024-09-21T10:01:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3130: finishing write tid 4 to smithi03385152-13 2024-09-21T10:01:23.586 INFO:tasks.rados.rados.0.smithi033.stdout:3130: finishing write tid 5 to smithi03385152-13 2024-09-21T10:01:23.586 INFO:tasks.rados.rados.0.smithi033.stdout:3130: finishing write tid 6 to smithi03385152-13 2024-09-21T10:01:23.586 INFO:tasks.rados.rados.0.smithi033.stdout:3132: finishing write tid 1 to smithi03385152-30 2024-09-21T10:01:23.586 INFO:tasks.rados.rados.0.smithi033.stdout:3130: finishing write tid 7 to smithi03385152-13 2024-09-21T10:01:23.586 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2340 (ObjNum 981 snap 286 seq_num 981) dirty exists 2024-09-21T10:01:23.586 INFO:tasks.rados.rados.0.smithi033.stdout:3130: left oid 13 (ObjNum 981 snap 286 seq_num 981) 2024-09-21T10:01:23.586 INFO:tasks.rados.rados.0.smithi033.stdout:3132: finishing write tid 2 to smithi03385152-30 2024-09-21T10:01:23.587 INFO:tasks.rados.rados.0.smithi033.stdout:3132: finishing write tid 3 to smithi03385152-30 2024-09-21T10:01:23.588 INFO:tasks.rados.rados.0.smithi033.stdout:3132: finishing write tid 4 to smithi03385152-30 2024-09-21T10:01:23.589 INFO:tasks.rados.rados.0.smithi033.stdout:3132: finishing write tid 5 to smithi03385152-30 2024-09-21T10:01:23.589 INFO:tasks.rados.rados.0.smithi033.stdout:3132: finishing write tid 6 to smithi03385152-30 2024-09-21T10:01:23.590 INFO:tasks.rados.rados.0.smithi033.stdout:3132: finishing write tid 7 to smithi03385152-30 2024-09-21T10:01:23.590 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2347 (ObjNum 982 snap 286 seq_num 982) dirty exists 2024-09-21T10:01:23.590 INFO:tasks.rados.rados.0.smithi033.stdout:3132: left oid 30 (ObjNum 982 snap 286 seq_num 982) 2024-09-21T10:01:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:23 smithi033 ceph-mon[98546]: pgmap v609: 121 pgs: 121 active+clean; 132 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T10:01:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:23 smithi033 ceph-mon[98546]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T10:01:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:23 smithi033 ceph-mon[98546]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T10:01:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:23 smithi033 ceph-mon[96568]: pgmap v609: 121 pgs: 121 active+clean; 132 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T10:01:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:23 smithi033 ceph-mon[96568]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T10:01:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:23 smithi033 ceph-mon[96568]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T10:01:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:23 smithi105 ceph-mon[86649]: pgmap v609: 121 pgs: 121 active+clean; 132 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-09-21T10:01:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:23 smithi105 ceph-mon[86649]: osdmap e653: 8 total, 8 up, 8 in 2024-09-21T10:01:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:23 smithi105 ceph-mon[86649]: osdmap e654: 8 total, 8 up, 8 in 2024-09-21T10:01:24.566 INFO:tasks.rados.rados.0.smithi033.stdout:3130: done (3 left) 2024-09-21T10:01:24.566 INFO:tasks.rados.rados.0.smithi033.stdout:3131: done (2 left) 2024-09-21T10:01:24.566 INFO:tasks.rados.rados.0.smithi033.stdout:3132: done (1 left) 2024-09-21T10:01:24.566 INFO:tasks.rados.rados.0.smithi033.stdout:3133: done (0 left) 2024-09-21T10:01:24.566 INFO:tasks.rados.rados.0.smithi033.stdout:3134: snap_remove snap 285 2024-09-21T10:01:25.569 INFO:tasks.rados.rados.0.smithi033.stdout:3134: done (0 left) 2024-09-21T10:01:25.569 INFO:tasks.rados.rados.0.smithi033.stdout:3135: setattr oid 40 current snap is 287 2024-09-21T10:01:25.570 INFO:tasks.rados.rados.0.smithi033.stdout:3136: read oid 41 snap 286 2024-09-21T10:01:25.570 INFO:tasks.rados.rados.0.smithi033.stdout:3136: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:25.571 INFO:tasks.rados.rados.0.smithi033.stdout:3137: rollback oid 34 current snap is 287 2024-09-21T10:01:25.571 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 286 2024-09-21T10:01:25.571 INFO:tasks.rados.rados.0.smithi033.stdout:3138: rmattr oid 13 current snap is 287 2024-09-21T10:01:25.571 INFO:tasks.rados.rados.0.smithi033.stdout:3138: done (3 left) 2024-09-21T10:01:25.571 INFO:tasks.rados.rados.0.smithi033.stdout:3139: setattr oid 17 current snap is 287 2024-09-21T10:01:25.571 INFO:tasks.rados.rados.0.smithi033.stdout:3140: setattr oid 21 current snap is 287 2024-09-21T10:01:25.574 INFO:tasks.rados.rados.0.smithi033.stdout:3141: read oid 44 snap 286 2024-09-21T10:01:25.574 INFO:tasks.rados.rados.0.smithi033.stdout:3141: expect deleted 2024-09-21T10:01:25.574 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 287 2024-09-21T10:01:25.574 INFO:tasks.rados.rados.0.smithi033.stdout:3142: seq_num 983 ranges {0=466944,466944=712704,1179648=434176,1613824=458752} 2024-09-21T10:01:25.577 INFO:tasks.rados.rados.0.smithi033.stdout:3142: writing smithi03385152-22 from 0 to 466944 tid 1 2024-09-21T10:01:25.582 INFO:tasks.rados.rados.0.smithi033.stdout:3142: writing smithi03385152-22 from 466944 to 1179648 tid 2 2024-09-21T10:01:25.585 INFO:tasks.rados.rados.0.smithi033.stdout:3142: writing smithi03385152-22 from 1179648 to 1613824 tid 3 2024-09-21T10:01:25.588 INFO:tasks.rados.rados.0.smithi033.stdout:3142: writing smithi03385152-22 from 1613824 to 2072576 tid 4 2024-09-21T10:01:25.588 INFO:tasks.rados.rados.0.smithi033.stdout:3143: copy_from oid 9 from oid 50 current snap is 287 2024-09-21T10:01:25.588 INFO:tasks.rados.rados.0.smithi033.stdout:3137: finishing rollback tid 1 to smithi03385152-34 2024-09-21T10:01:25.588 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 0 (ObjNum 33 snap 0 seq_num 1414816100) dirty dne 2024-09-21T10:01:25.588 INFO:tasks.rados.rados.0.smithi033.stdout:3137: done (7 left) 2024-09-21T10:01:25.588 INFO:tasks.rados.rados.0.smithi033.stdout:3144: delete oid 4 current snap is 287 2024-09-21T10:01:25.588 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 1637 (ObjNum 947 snap 279 seq_num 947) dirty exists 2024-09-21T10:01:25.588 INFO:tasks.rados.rados.0.smithi033.stdout:3136: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:25.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1809 (ObjNum 289 snap 0 seq_num 1) dirty exists 2024-09-21T10:01:25.589 INFO:tasks.rados.rados.0.smithi033.stdout:3142: finishing write tid 1 to smithi03385152-22 2024-09-21T10:01:25.590 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 1621 (ObjNum 909 snap 263 seq_num 909) dirty exists 2024-09-21T10:01:25.590 INFO:tasks.rados.rados.0.smithi033.stdout:3142: finishing write tid 2 to smithi03385152-22 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:3135: done (7 left) 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:3136: done (6 left) 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:3139: done (5 left) 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:3140: done (4 left) 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:3141: done (3 left) 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:3144: done (2 left) 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:3145: copy_from oid 2 from oid 11 current snap is 287 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 287 2024-09-21T10:01:25.591 INFO:tasks.rados.rados.0.smithi033.stdout:3146: seq_num 985 ranges {5013504=499712,5513216=483328,5996544=450560,6447104=581632,7028736=90112} 2024-09-21T10:01:25.621 INFO:tasks.rados.rados.0.smithi033.stdout:3146: writing smithi03385152-13 from 5013504 to 5513216 tid 1 2024-09-21T10:01:25.624 INFO:tasks.rados.rados.0.smithi033.stdout:3146: writing smithi03385152-13 from 5513216 to 5996544 tid 2 2024-09-21T10:01:25.627 INFO:tasks.rados.rados.0.smithi033.stdout:3146: writing smithi03385152-13 from 5996544 to 6447104 tid 3 2024-09-21T10:01:25.630 INFO:tasks.rados.rados.0.smithi033.stdout:3146: writing smithi03385152-13 from 6447104 to 7028736 tid 4 2024-09-21T10:01:25.631 INFO:tasks.rados.rados.0.smithi033.stdout:3146: writing smithi03385152-13 from 7028736 to 7118848 tid 5 2024-09-21T10:01:25.631 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 287 2024-09-21T10:01:25.631 INFO:tasks.rados.rados.0.smithi033.stdout:3142: finishing write tid 3 to smithi03385152-22 2024-09-21T10:01:25.631 INFO:tasks.rados.rados.0.smithi033.stdout:3147: seq_num 986 ranges {114688=516096,630784=770048,1400832=655360,2056192=180224} 2024-09-21T10:01:25.634 INFO:tasks.rados.rados.0.smithi033.stdout:3147: writing smithi03385152-1 from 114688 to 630784 tid 1 2024-09-21T10:01:25.639 INFO:tasks.rados.rados.0.smithi033.stdout:3147: writing smithi03385152-1 from 630784 to 1400832 tid 2 2024-09-21T10:01:25.643 INFO:tasks.rados.rados.0.smithi033.stdout:3147: writing smithi03385152-1 from 1400832 to 2056192 tid 3 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3147: writing smithi03385152-1 from 2056192 to 2236416 tid 4 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3148: copy_from oid 47 from oid 10 current snap is 287 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3142: finishing write tid 4 to smithi03385152-22 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3142: finishing write tid 5 to smithi03385152-22 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3145: finishing copy_from to smithi03385152-2 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3145: got expected ENOENT (src dne) 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3142: finishing write tid 6 to smithi03385152-22 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 1866 (ObjNum 983 snap 287 seq_num 983) dirty exists 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3142: left oid 22 (ObjNum 983 snap 287 seq_num 983) 2024-09-21T10:01:25.645 INFO:tasks.rados.rados.0.smithi033.stdout:3145: finishing copy_from racing read to smithi03385152-2 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3143: finishing copy_from racing read to smithi03385152-9 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3143: finishing copy_from to smithi03385152-9 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 1800 (ObjNum 965 snap 283 seq_num 965) dirty exists 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3146: finishing write tid 1 to smithi03385152-13 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3146: finishing write tid 2 to smithi03385152-13 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3146: finishing write tid 3 to smithi03385152-13 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3142: done (5 left) 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3143: done (4 left) 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3145: done (3 left) 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3149: rollback oid 18 current snap is 287 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 286 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3146: finishing write tid 4 to smithi03385152-13 2024-09-21T10:01:25.646 INFO:tasks.rados.rados.0.smithi033.stdout:3146: finishing write tid 5 to smithi03385152-13 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3146: finishing write tid 6 to smithi03385152-13 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3146: finishing write tid 7 to smithi03385152-13 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2358 (ObjNum 985 snap 287 seq_num 985) dirty exists 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3146: left oid 13 (ObjNum 985 snap 287 seq_num 985) 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3147: finishing write tid 1 to smithi03385152-1 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3146: done (3 left) 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3150: read oid 48 snap -1 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3150: expect (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3151: snap_create 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3147: finishing write tid 2 to smithi03385152-1 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3148: finishing copy_from to smithi03385152-47 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3148: got expected ENOENT (src dne) 2024-09-21T10:01:25.647 INFO:tasks.rados.rados.0.smithi033.stdout:3148: finishing copy_from racing read to smithi03385152-47 2024-09-21T10:01:25.648 INFO:tasks.rados.rados.0.smithi033.stdout:3149: finishing rollback tid 0 to smithi03385152-18 2024-09-21T10:01:25.648 INFO:tasks.rados.rados.0.smithi033.stdout:3147: finishing write tid 3 to smithi03385152-1 2024-09-21T10:01:25.648 INFO:tasks.rados.rados.0.smithi033.stdout:3149: finishing rollback tid 1 to smithi03385152-18 2024-09-21T10:01:25.648 INFO:tasks.rados.rados.0.smithi033.stdout:3147: finishing write tid 4 to smithi03385152-1 2024-09-21T10:01:25.648 INFO:tasks.rados.rados.0.smithi033.stdout:3149: finishing rollback tid 2 to smithi03385152-18 2024-09-21T10:01:25.648 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1027 (ObjNum 966 snap 283 seq_num 966) dirty exists 2024-09-21T10:01:25.648 INFO:tasks.rados.rados.0.smithi033.stdout:3147: finishing write tid 5 to smithi03385152-1 2024-09-21T10:01:25.650 INFO:tasks.rados.rados.0.smithi033.stdout:3147: finishing write tid 6 to smithi03385152-1 2024-09-21T10:01:25.650 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1508 (ObjNum 986 snap 287 seq_num 986) dirty exists 2024-09-21T10:01:25.650 INFO:tasks.rados.rados.0.smithi033.stdout:3147: left oid 1 (ObjNum 986 snap 287 seq_num 986) 2024-09-21T10:01:25.685 INFO:tasks.rados.rados.0.smithi033.stdout:3150: expect (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:01:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:25 smithi033 ceph-mon[98546]: pgmap v612: 121 pgs: 121 active+clean; 115 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T10:01:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:25 smithi033 ceph-mon[98546]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T10:01:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:25 smithi033 ceph-mon[96568]: pgmap v612: 121 pgs: 121 active+clean; 115 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T10:01:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:25 smithi033 ceph-mon[96568]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T10:01:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:25 smithi105 ceph-mon[86649]: pgmap v612: 121 pgs: 121 active+clean; 115 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.8 MiB/s wr, 17 op/s 2024-09-21T10:01:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:25 smithi105 ceph-mon[86649]: osdmap e655: 8 total, 8 up, 8 in 2024-09-21T10:01:26.581 INFO:tasks.rados.rados.0.smithi033.stdout:3147: done (4 left) 2024-09-21T10:01:26.581 INFO:tasks.rados.rados.0.smithi033.stdout:3148: done (3 left) 2024-09-21T10:01:26.581 INFO:tasks.rados.rados.0.smithi033.stdout:3149: done (2 left) 2024-09-21T10:01:26.581 INFO:tasks.rados.rados.0.smithi033.stdout:3150: done (1 left) 2024-09-21T10:01:26.581 INFO:tasks.rados.rados.0.smithi033.stdout:3151: done (0 left) 2024-09-21T10:01:26.581 INFO:tasks.rados.rados.0.smithi033.stdout:3152: rmattr oid 43 current snap is 288 2024-09-21T10:01:26.581 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 288 2024-09-21T10:01:26.581 INFO:tasks.rados.rados.0.smithi033.stdout:3153: seq_num 987 ranges {0=532480,532480=647168,1179648=417792,1597440=409600,2007040=131072} 2024-09-21T10:01:26.584 INFO:tasks.rados.rados.0.smithi033.stdout:3153: writing smithi03385152-49 from 0 to 532480 tid 1 2024-09-21T10:01:26.587 INFO:tasks.rados.rados.0.smithi033.stdout:3153: writing smithi03385152-49 from 532480 to 1179648 tid 2 2024-09-21T10:01:26.589 INFO:tasks.rados.rados.0.smithi033.stdout:3153: writing smithi03385152-49 from 1179648 to 1597440 tid 3 2024-09-21T10:01:26.592 INFO:tasks.rados.rados.0.smithi033.stdout:3153: writing smithi03385152-49 from 1597440 to 2007040 tid 4 2024-09-21T10:01:26.593 INFO:tasks.rados.rados.0.smithi033.stdout:3153: writing smithi03385152-49 from 2007040 to 2138112 tid 5 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3154: read oid 14 snap -1 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3154: expect (ObjNum 952 snap 279 seq_num 952) 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3155: copy_from oid 26 from oid 15 current snap is 288 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1532 (ObjNum 949 snap 279 seq_num 949) dirty exists 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3153: finishing write tid 1 to smithi03385152-49 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3153: finishing write tid 2 to smithi03385152-49 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3152: done (3 left) 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3156: copy_from oid 16 from oid 35 current snap is 288 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3153: finishing write tid 3 to smithi03385152-49 2024-09-21T10:01:26.594 INFO:tasks.rados.rados.0.smithi033.stdout:3157: snap_remove snap 286 2024-09-21T10:01:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:26 smithi033 ceph-mon[98546]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T10:01:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:26 smithi033 ceph-mon[96568]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T10:01:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:26 smithi105 ceph-mon[86649]: osdmap e656: 8 total, 8 up, 8 in 2024-09-21T10:01:27.586 INFO:tasks.rados.rados.0.smithi033.stdout:3153: finishing write tid 4 to smithi03385152-49 2024-09-21T10:01:27.587 INFO:tasks.rados.rados.0.smithi033.stdout:3153: finishing write tid 5 to smithi03385152-49 2024-09-21T10:01:27.587 INFO:tasks.rados.rados.0.smithi033.stdout:3153: finishing write tid 6 to smithi03385152-49 2024-09-21T10:01:27.587 INFO:tasks.rados.rados.0.smithi033.stdout:3153: finishing write tid 7 to smithi03385152-49 2024-09-21T10:01:27.587 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 1643 (ObjNum 987 snap 288 seq_num 987) dirty exists 2024-09-21T10:01:27.587 INFO:tasks.rados.rados.0.smithi033.stdout:3153: left oid 49 (ObjNum 987 snap 288 seq_num 987) 2024-09-21T10:01:27.587 INFO:tasks.rados.rados.0.smithi033.stdout:3156: finishing copy_from to smithi03385152-16 2024-09-21T10:01:27.587 INFO:tasks.rados.rados.0.smithi033.stdout:3156: got expected ENOENT (src dne) 2024-09-21T10:01:27.587 INFO:tasks.rados.rados.0.smithi033.stdout:3154: expect (ObjNum 952 snap 279 seq_num 952) 2024-09-21T10:01:27.630 INFO:tasks.rados.rados.0.smithi033.stdout:3156: finishing copy_from racing read to smithi03385152-16 2024-09-21T10:01:27.630 INFO:tasks.rados.rados.0.smithi033.stdout:3155: finishing copy_from racing read to smithi03385152-26 2024-09-21T10:01:27.630 INFO:tasks.rados.rados.0.smithi033.stdout:3155: finishing copy_from to smithi03385152-26 2024-09-21T10:01:27.630 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 1502 (ObjNum 962 snap 281 seq_num 962) dirty exists 2024-09-21T10:01:27.630 INFO:tasks.rados.rados.0.smithi033.stdout:3153: done (4 left) 2024-09-21T10:01:27.630 INFO:tasks.rados.rados.0.smithi033.stdout:3154: done (3 left) 2024-09-21T10:01:27.631 INFO:tasks.rados.rados.0.smithi033.stdout:3155: done (2 left) 2024-09-21T10:01:27.631 INFO:tasks.rados.rados.0.smithi033.stdout:3156: done (1 left) 2024-09-21T10:01:27.631 INFO:tasks.rados.rados.0.smithi033.stdout:3157: done (0 left) 2024-09-21T10:01:27.631 INFO:tasks.rados.rados.0.smithi033.stdout:3158: rmattr oid 40 current snap is 288 2024-09-21T10:01:27.631 INFO:tasks.rados.rados.0.smithi033.stdout:3159: snap_remove snap 287 2024-09-21T10:01:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:27 smithi033 ceph-mon[98546]: pgmap v615: 121 pgs: 121 active+clean; 108 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.0 MiB/s wr, 28 op/s 2024-09-21T10:01:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:27 smithi033 ceph-mon[98546]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T10:01:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:27 smithi033 ceph-mon[96568]: pgmap v615: 121 pgs: 121 active+clean; 108 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.0 MiB/s wr, 28 op/s 2024-09-21T10:01:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:27 smithi033 ceph-mon[96568]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T10:01:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:27 smithi105 ceph-mon[86649]: pgmap v615: 121 pgs: 121 active+clean; 108 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.2 KiB/s rd, 7.0 MiB/s wr, 28 op/s 2024-09-21T10:01:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:27 smithi105 ceph-mon[86649]: osdmap e657: 8 total, 8 up, 8 in 2024-09-21T10:01:28.547 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1810 (ObjNum 49 snap 0 seq_num 1497457549) dirty exists 2024-09-21T10:01:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:3158: done (1 left) 2024-09-21T10:01:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:3159: done (0 left) 2024-09-21T10:01:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:3160: rollback oid 46 current snap is 288 2024-09-21T10:01:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:3160: done (0 left) 2024-09-21T10:01:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:3161: read oid 31 snap -1 2024-09-21T10:01:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:3161: expect (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:01:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 288 2024-09-21T10:01:28.548 INFO:tasks.rados.rados.0.smithi033.stdout:3162: seq_num 988 ranges {0=557056,557056=532480,1089536=622592,1712128=450560} 2024-09-21T10:01:28.550 INFO:tasks.rados.rados.0.smithi033.stdout:3162: writing smithi03385152-39 from 0 to 557056 tid 1 2024-09-21T10:01:28.554 INFO:tasks.rados.rados.0.smithi033.stdout:3162: writing smithi03385152-39 from 557056 to 1089536 tid 2 2024-09-21T10:01:28.558 INFO:tasks.rados.rados.0.smithi033.stdout:3162: writing smithi03385152-39 from 1089536 to 1712128 tid 3 2024-09-21T10:01:28.562 INFO:tasks.rados.rados.0.smithi033.stdout:3162: writing smithi03385152-39 from 1712128 to 2162688 tid 4 2024-09-21T10:01:28.562 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 288 2024-09-21T10:01:28.562 INFO:tasks.rados.rados.0.smithi033.stdout:3163: seq_num 989 ranges {1753088=573440,2326528=466944,2793472=434176,3227648=638976,3866624=65536} 2024-09-21T10:01:28.575 INFO:tasks.rados.rados.0.smithi033.stdout:3163: writing smithi03385152-20 from 1753088 to 2326528 tid 1 2024-09-21T10:01:28.578 INFO:tasks.rados.rados.0.smithi033.stdout:3163: writing smithi03385152-20 from 2326528 to 2793472 tid 2 2024-09-21T10:01:28.580 INFO:tasks.rados.rados.0.smithi033.stdout:3163: writing smithi03385152-20 from 2793472 to 3227648 tid 3 2024-09-21T10:01:28.584 INFO:tasks.rados.rados.0.smithi033.stdout:3163: writing smithi03385152-20 from 3227648 to 3866624 tid 4 2024-09-21T10:01:28.585 INFO:tasks.rados.rados.0.smithi033.stdout:3163: writing smithi03385152-20 from 3866624 to 3932160 tid 5 2024-09-21T10:01:28.585 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 288 2024-09-21T10:01:28.585 INFO:tasks.rados.rados.0.smithi033.stdout:3164: seq_num 990 ranges {0=589824,589824=753664,1343488=638976,1982464=212992} 2024-09-21T10:01:28.589 INFO:tasks.rados.rados.0.smithi033.stdout:3164: writing smithi03385152-19 from 0 to 589824 tid 1 2024-09-21T10:01:28.594 INFO:tasks.rados.rados.0.smithi033.stdout:3164: writing smithi03385152-19 from 589824 to 1343488 tid 2 2024-09-21T10:01:28.598 INFO:tasks.rados.rados.0.smithi033.stdout:3164: writing smithi03385152-19 from 1343488 to 1982464 tid 3 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3164: writing smithi03385152-19 from 1982464 to 2195456 tid 4 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3165: read oid 43 snap -1 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3165: expect (ObjNum 949 snap 279 seq_num 949) 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3166: read oid 4 snap -1 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3166: expect deleted 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3167: read oid 50 snap -1 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3167: expect (ObjNum 965 snap 283 seq_num 965) 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3168: rollback oid 15 current snap is 288 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3168: done (7 left) 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 288 2024-09-21T10:01:28.600 INFO:tasks.rados.rados.0.smithi033.stdout:3169: seq_num 991 ranges {1081344=606208,1687552=638976,2326528=794624,3121152=172032} 2024-09-21T10:01:28.609 INFO:tasks.rados.rados.0.smithi033.stdout:3169: writing smithi03385152-5 from 1081344 to 1687552 tid 1 2024-09-21T10:01:28.614 INFO:tasks.rados.rados.0.smithi033.stdout:3169: writing smithi03385152-5 from 1687552 to 2326528 tid 2 2024-09-21T10:01:28.619 INFO:tasks.rados.rados.0.smithi033.stdout:3169: writing smithi03385152-5 from 2326528 to 3121152 tid 3 2024-09-21T10:01:28.621 INFO:tasks.rados.rados.0.smithi033.stdout:3169: writing smithi03385152-5 from 3121152 to 3293184 tid 4 2024-09-21T10:01:28.621 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 288 2024-09-21T10:01:28.621 INFO:tasks.rados.rados.0.smithi033.stdout:3162: finishing write tid 1 to smithi03385152-39 2024-09-21T10:01:28.621 INFO:tasks.rados.rados.0.smithi033.stdout:3170: seq_num 992 ranges {1695744=622592,2318336=524288,2842624=606208,3448832=475136} 2024-09-21T10:01:28.635 INFO:tasks.rados.rados.0.smithi033.stdout:3170: writing smithi03385152-8 from 1695744 to 2318336 tid 1 2024-09-21T10:01:28.638 INFO:tasks.rados.rados.0.smithi033.stdout:3170: writing smithi03385152-8 from 2318336 to 2842624 tid 2 2024-09-21T10:01:28.642 INFO:tasks.rados.rados.0.smithi033.stdout:3170: writing smithi03385152-8 from 2842624 to 3448832 tid 3 2024-09-21T10:01:28.646 INFO:tasks.rados.rados.0.smithi033.stdout:3170: writing smithi03385152-8 from 3448832 to 3923968 tid 4 2024-09-21T10:01:28.646 INFO:tasks.rados.rados.0.smithi033.stdout:3171: copy_from oid 18 from oid 14 current snap is 288 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3162: finishing write tid 2 to smithi03385152-39 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3162: finishing write tid 3 to smithi03385152-39 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3162: finishing write tid 4 to smithi03385152-39 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3162: finishing write tid 5 to smithi03385152-39 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3162: finishing write tid 6 to smithi03385152-39 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 1060 (ObjNum 988 snap 288 seq_num 988) dirty exists 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3162: left oid 39 (ObjNum 988 snap 288 seq_num 988) 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3163: finishing write tid 1 to smithi03385152-20 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3163: finishing write tid 2 to smithi03385152-20 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3163: finishing write tid 3 to smithi03385152-20 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3163: finishing write tid 4 to smithi03385152-20 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3162: done (9 left) 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3172: read oid 49 snap -1 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3172: expect (ObjNum 987 snap 288 seq_num 987) 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 288 2024-09-21T10:01:28.647 INFO:tasks.rados.rados.0.smithi033.stdout:3163: finishing write tid 5 to smithi03385152-20 2024-09-21T10:01:28.648 INFO:tasks.rados.rados.0.smithi033.stdout:3163: finishing write tid 6 to smithi03385152-20 2024-09-21T10:01:28.648 INFO:tasks.rados.rados.0.smithi033.stdout:3164: finishing write tid 1 to smithi03385152-19 2024-09-21T10:01:28.648 INFO:tasks.rados.rados.0.smithi033.stdout:3163: finishing write tid 7 to smithi03385152-20 2024-09-21T10:01:28.648 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1627 (ObjNum 989 snap 288 seq_num 989) dirty exists 2024-09-21T10:01:28.648 INFO:tasks.rados.rados.0.smithi033.stdout:3163: left oid 20 (ObjNum 989 snap 288 seq_num 989) 2024-09-21T10:01:28.648 INFO:tasks.rados.rados.0.smithi033.stdout:3164: finishing write tid 2 to smithi03385152-19 2024-09-21T10:01:28.648 INFO:tasks.rados.rados.0.smithi033.stdout:3161: expect (ObjNum 909 snap 263 seq_num 909) 2024-09-21T10:01:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:28 smithi033 ceph-mon[98546]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T10:01:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:28 smithi033 ceph-mon[98546]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T10:01:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:28 smithi033 ceph-mon[96568]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T10:01:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:28 smithi033 ceph-mon[96568]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T10:01:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:28 smithi105 ceph-mon[86649]: osdmap e658: 8 total, 8 up, 8 in 2024-09-21T10:01:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:28 smithi105 ceph-mon[86649]: osdmap e659: 8 total, 8 up, 8 in 2024-09-21T10:01:29.443 INFO:tasks.rados.rados.0.smithi033.stdout:3164: finishing write tid 3 to smithi03385152-19 2024-09-21T10:01:29.444 INFO:tasks.rados.rados.0.smithi033.stdout:3164: finishing write tid 4 to smithi03385152-19 2024-09-21T10:01:29.444 INFO:tasks.rados.rados.0.smithi033.stdout:3164: finishing write tid 5 to smithi03385152-19 2024-09-21T10:01:29.444 INFO:tasks.rados.rados.0.smithi033.stdout:3164: finishing write tid 6 to smithi03385152-19 2024-09-21T10:01:29.446 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2258 (ObjNum 990 snap 288 seq_num 990) dirty exists 2024-09-21T10:01:29.446 INFO:tasks.rados.rados.0.smithi033.stdout:3164: left oid 19 (ObjNum 990 snap 288 seq_num 990) 2024-09-21T10:01:29.446 INFO:tasks.rados.rados.0.smithi033.stdout:3169: finishing write tid 1 to smithi03385152-5 2024-09-21T10:01:29.446 INFO:tasks.rados.rados.0.smithi033.stdout:3167: expect (ObjNum 965 snap 283 seq_num 965) 2024-09-21T10:01:29.613 INFO:tasks.rados.rados.0.smithi033.stdout:3169: finishing write tid 2 to smithi03385152-5 2024-09-21T10:01:29.614 INFO:tasks.rados.rados.0.smithi033.stdout:3169: finishing write tid 3 to smithi03385152-5 2024-09-21T10:01:29.614 INFO:tasks.rados.rados.0.smithi033.stdout:3173: seq_num 993 ranges {917504=638976,1556480=409600,1966080=409600,2375680=688128,3063808=98304} 2024-09-21T10:01:29.622 INFO:tasks.rados.rados.0.smithi033.stdout:3173: writing smithi03385152-45 from 917504 to 1556480 tid 1 2024-09-21T10:01:29.624 INFO:tasks.rados.rados.0.smithi033.stdout:3173: writing smithi03385152-45 from 1556480 to 1966080 tid 2 2024-09-21T10:01:29.626 INFO:tasks.rados.rados.0.smithi033.stdout:3173: writing smithi03385152-45 from 1966080 to 2375680 tid 3 2024-09-21T10:01:29.630 INFO:tasks.rados.rados.0.smithi033.stdout:3173: writing smithi03385152-45 from 2375680 to 3063808 tid 4 2024-09-21T10:01:29.631 INFO:tasks.rados.rados.0.smithi033.stdout:3173: writing smithi03385152-45 from 3063808 to 3162112 tid 5 2024-09-21T10:01:29.631 INFO:tasks.rados.rados.0.smithi033.stdout:3161: done (10 left) 2024-09-21T10:01:29.631 INFO:tasks.rados.rados.0.smithi033.stdout:3163: done (9 left) 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3164: done (8 left) 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3166: done (7 left) 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3167: done (6 left) 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 288 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3169: finishing write tid 4 to smithi03385152-5 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3169: finishing write tid 5 to smithi03385152-5 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3169: finishing write tid 6 to smithi03385152-5 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 1065 (ObjNum 991 snap 288 seq_num 991) dirty exists 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3169: left oid 5 (ObjNum 991 snap 288 seq_num 991) 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3170: finishing write tid 1 to smithi03385152-8 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3170: finishing write tid 2 to smithi03385152-8 2024-09-21T10:01:29.632 INFO:tasks.rados.rados.0.smithi033.stdout:3165: expect (ObjNum 949 snap 279 seq_num 949) 2024-09-21T10:01:29.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:29 smithi105 ceph-mon[86649]: pgmap v618: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 110 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-21T10:01:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:29 smithi033 ceph-mon[98546]: pgmap v618: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 110 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-21T10:01:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:29 smithi033 ceph-mon[96568]: pgmap v618: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 110 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 KiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-21T10:01:30.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:01:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:01:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:3170: finishing write tid 3 to smithi03385152-8 2024-09-21T10:01:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:3170: finishing write tid 4 to smithi03385152-8 2024-09-21T10:01:30.610 INFO:tasks.rados.rados.0.smithi033.stdout:3174: seq_num 994 ranges {0=655360,655360=688128,1343488=622592,1966080=294912} 2024-09-21T10:01:30.613 INFO:tasks.rados.rados.0.smithi033.stdout:3174: writing smithi03385152-10 from 0 to 655360 tid 1 2024-09-21T10:01:30.618 INFO:tasks.rados.rados.0.smithi033.stdout:3174: writing smithi03385152-10 from 655360 to 1343488 tid 2 2024-09-21T10:01:30.621 INFO:tasks.rados.rados.0.smithi033.stdout:3174: writing smithi03385152-10 from 1343488 to 1966080 tid 3 2024-09-21T10:01:30.624 INFO:tasks.rados.rados.0.smithi033.stdout:3174: writing smithi03385152-10 from 1966080 to 2260992 tid 4 2024-09-21T10:01:30.624 INFO:tasks.rados.rados.0.smithi033.stdout:3165: done (6 left) 2024-09-21T10:01:30.624 INFO:tasks.rados.rados.0.smithi033.stdout:3169: done (5 left) 2024-09-21T10:01:30.624 INFO:tasks.rados.rados.0.smithi033.stdout:3175: snap_create 2024-09-21T10:01:30.624 INFO:tasks.rados.rados.0.smithi033.stdout:3170: finishing write tid 5 to smithi03385152-8 2024-09-21T10:01:30.624 INFO:tasks.rados.rados.0.smithi033.stdout:3170: finishing write tid 6 to smithi03385152-8 2024-09-21T10:01:30.624 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 1513 (ObjNum 992 snap 288 seq_num 992) dirty exists 2024-09-21T10:01:30.624 INFO:tasks.rados.rados.0.smithi033.stdout:3170: left oid 8 (ObjNum 992 snap 288 seq_num 992) 2024-09-21T10:01:30.625 INFO:tasks.rados.rados.0.smithi033.stdout:3171: finishing copy_from racing read to smithi03385152-18 2024-09-21T10:01:30.625 INFO:tasks.rados.rados.0.smithi033.stdout:3171: finishing copy_from to smithi03385152-18 2024-09-21T10:01:30.625 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1816 (ObjNum 952 snap 279 seq_num 952) dirty exists 2024-09-21T10:01:30.625 INFO:tasks.rados.rados.0.smithi033.stdout:3172: expect (ObjNum 987 snap 288 seq_num 987) 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3173: finishing write tid 1 to smithi03385152-45 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3173: finishing write tid 2 to smithi03385152-45 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3173: finishing write tid 3 to smithi03385152-45 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3173: finishing write tid 4 to smithi03385152-45 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3173: finishing write tid 5 to smithi03385152-45 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3173: finishing write tid 6 to smithi03385152-45 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3173: finishing write tid 7 to smithi03385152-45 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1874 (ObjNum 993 snap 288 seq_num 993) dirty exists 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3173: left oid 45 (ObjNum 993 snap 288 seq_num 993) 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3174: finishing write tid 1 to smithi03385152-10 2024-09-21T10:01:30.681 INFO:tasks.rados.rados.0.smithi033.stdout:3174: finishing write tid 2 to smithi03385152-10 2024-09-21T10:01:30.682 INFO:tasks.rados.rados.0.smithi033.stdout:3174: finishing write tid 3 to smithi03385152-10 2024-09-21T10:01:30.682 INFO:tasks.rados.rados.0.smithi033.stdout:3174: finishing write tid 4 to smithi03385152-10 2024-09-21T10:01:30.682 INFO:tasks.rados.rados.0.smithi033.stdout:3174: finishing write tid 5 to smithi03385152-10 2024-09-21T10:01:30.682 INFO:tasks.rados.rados.0.smithi033.stdout:3174: finishing write tid 6 to smithi03385152-10 2024-09-21T10:01:30.682 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 2263 (ObjNum 994 snap 288 seq_num 994) dirty exists 2024-09-21T10:01:30.682 INFO:tasks.rados.rados.0.smithi033.stdout:3174: left oid 10 (ObjNum 994 snap 288 seq_num 994) 2024-09-21T10:01:31.600 INFO:tasks.rados.rados.0.smithi033.stdout:3170: done (5 left) 2024-09-21T10:01:31.600 INFO:tasks.rados.rados.0.smithi033.stdout:3171: done (4 left) 2024-09-21T10:01:31.600 INFO:tasks.rados.rados.0.smithi033.stdout:3172: done (3 left) 2024-09-21T10:01:31.601 INFO:tasks.rados.rados.0.smithi033.stdout:3173: done (2 left) 2024-09-21T10:01:31.601 INFO:tasks.rados.rados.0.smithi033.stdout:3174: done (1 left) 2024-09-21T10:01:31.601 INFO:tasks.rados.rados.0.smithi033.stdout:3175: done (0 left) 2024-09-21T10:01:31.601 INFO:tasks.rados.rados.0.smithi033.stdout:3176: read oid 28 snap -1 2024-09-21T10:01:31.601 INFO:tasks.rados.rados.0.smithi033.stdout:3176: expect (ObjNum 914 snap 265 seq_num 914) 2024-09-21T10:01:31.601 INFO:tasks.rados.rados.0.smithi033.stdout:3177: read oid 20 snap -1 2024-09-21T10:01:31.601 INFO:tasks.rados.rados.0.smithi033.stdout:3177: expect (ObjNum 989 snap 288 seq_num 989) 2024-09-21T10:01:31.601 INFO:tasks.rados.rados.0.smithi033.stdout:3178: snap_remove snap 288 2024-09-21T10:01:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:01:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:01:31] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-09-21T10:01:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:31 smithi033 ceph-mon[98546]: pgmap v620: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 114 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.4 MiB/s wr, 29 op/s 2024-09-21T10:01:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:31 smithi033 ceph-mon[96568]: pgmap v620: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 114 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.4 MiB/s wr, 29 op/s 2024-09-21T10:01:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:31 smithi105 ceph-mon[86649]: pgmap v620: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 114 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.4 MiB/s wr, 29 op/s 2024-09-21T10:01:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:3178: done (2 left) 2024-09-21T10:01:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:3179: snap_create 2024-09-21T10:01:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:3176: expect (ObjNum 914 snap 265 seq_num 914) 2024-09-21T10:01:32.639 INFO:tasks.rados.rados.0.smithi033.stdout:3177: expect (ObjNum 989 snap 288 seq_num 989) 2024-09-21T10:01:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:32 smithi033 ceph-mon[98546]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T10:01:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:32 smithi033 ceph-mon[96568]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T10:01:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:32 smithi105 ceph-mon[86649]: osdmap e660: 8 total, 8 up, 8 in 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3176: done (2 left) 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3177: done (1 left) 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3179: done (0 left) 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3180: read oid 39 snap -1 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3180: expect (ObjNum 988 snap 288 seq_num 988) 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3181: read oid 46 snap -1 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3181: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3182: copy_from oid 32 from oid 35 current snap is 290 2024-09-21T10:01:33.544 INFO:tasks.rados.rados.0.smithi033.stdout:3183: copy_from oid 4 from oid 6 current snap is 290 2024-09-21T10:01:33.545 INFO:tasks.rados.rados.0.smithi033.stdout:3184: snap_remove snap 289 2024-09-21T10:01:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:33 smithi033 ceph-mon[98546]: pgmap v622: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 121 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-09-21T10:01:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:33 smithi033 ceph-mon[98546]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T10:01:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:33 smithi033 ceph-mon[98546]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T10:01:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:33 smithi033 ceph-mon[96568]: pgmap v622: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 121 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-09-21T10:01:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:33 smithi033 ceph-mon[96568]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T10:01:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:33 smithi033 ceph-mon[96568]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T10:01:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:33 smithi105 ceph-mon[86649]: pgmap v622: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 121 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.4 MiB/s wr, 21 op/s 2024-09-21T10:01:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:33 smithi105 ceph-mon[86649]: osdmap e661: 8 total, 8 up, 8 in 2024-09-21T10:01:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:33 smithi105 ceph-mon[86649]: osdmap e662: 8 total, 8 up, 8 in 2024-09-21T10:01:34.547 INFO:tasks.rados.rados.0.smithi033.stdout:3184: done (4 left) 2024-09-21T10:01:34.547 INFO:tasks.rados.rados.0.smithi033.stdout:3185: read oid 41 snap -1 2024-09-21T10:01:34.548 INFO:tasks.rados.rados.0.smithi033.stdout:3185: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:34.548 INFO:tasks.rados.rados.0.smithi033.stdout:3186: copy_from oid 8 from oid 36 current snap is 290 2024-09-21T10:01:34.548 INFO:tasks.rados.rados.0.smithi033.stdout:3182: finishing copy_from to smithi03385152-32 2024-09-21T10:01:34.548 INFO:tasks.rados.rados.0.smithi033.stdout:3182: got expected ENOENT (src dne) 2024-09-21T10:01:34.548 INFO:tasks.rados.rados.0.smithi033.stdout:3182: finishing copy_from racing read to smithi03385152-32 2024-09-21T10:01:34.548 INFO:tasks.rados.rados.0.smithi033.stdout:3181: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:34.550 INFO:tasks.rados.rados.0.smithi033.stdout:3180: expect (ObjNum 988 snap 288 seq_num 988) 2024-09-21T10:01:34.612 INFO:tasks.rados.rados.0.smithi033.stdout:3183: finishing copy_from racing read to smithi03385152-4 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3183: finishing copy_from to smithi03385152-4 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3180: done (5 left) 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3181: done (4 left) 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3182: done (3 left) 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3183: done (2 left) 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3187: snap_create 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3186: finishing copy_from to smithi03385152-8 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3186: got expected ENOENT (src dne) 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3186: finishing copy_from racing read to smithi03385152-8 2024-09-21T10:01:34.613 INFO:tasks.rados.rados.0.smithi033.stdout:3185: expect (ObjNum 863 snap 253 seq_num 863) 2024-09-21T10:01:35.621 INFO:tasks.rados.rados.0.smithi033.stdout:3185: done (2 left) 2024-09-21T10:01:35.621 INFO:tasks.rados.rados.0.smithi033.stdout:3186: done (1 left) 2024-09-21T10:01:35.621 INFO:tasks.rados.rados.0.smithi033.stdout:3187: done (0 left) 2024-09-21T10:01:35.621 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 291 2024-09-21T10:01:35.621 INFO:tasks.rados.rados.0.smithi033.stdout:3188: seq_num 995 ranges {0=671744,671744=573440,1245184=778240,2023424=253952} 2024-09-21T10:01:35.624 INFO:tasks.rados.rados.0.smithi033.stdout:3188: writing smithi03385152-47 from 0 to 671744 tid 1 2024-09-21T10:01:35.628 INFO:tasks.rados.rados.0.smithi033.stdout:3188: writing smithi03385152-47 from 671744 to 1245184 tid 2 2024-09-21T10:01:35.633 INFO:tasks.rados.rados.0.smithi033.stdout:3188: writing smithi03385152-47 from 1245184 to 2023424 tid 3 2024-09-21T10:01:35.634 INFO:tasks.rados.rados.0.smithi033.stdout:3188: writing smithi03385152-47 from 2023424 to 2277376 tid 4 2024-09-21T10:01:35.634 INFO:tasks.rados.rados.0.smithi033.stdout:3189: snap_remove snap 290 2024-09-21T10:01:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:35 smithi033 ceph-mon[96568]: pgmap v625: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 5.1 MiB/s wr, 25 op/s 2024-09-21T10:01:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:35 smithi033 ceph-mon[96568]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T10:01:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:35 smithi033 ceph-mon[98546]: pgmap v625: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 5.1 MiB/s wr, 25 op/s 2024-09-21T10:01:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:35 smithi033 ceph-mon[98546]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T10:01:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:35 smithi105 ceph-mon[86649]: pgmap v625: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 5.1 MiB/s wr, 25 op/s 2024-09-21T10:01:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:35 smithi105 ceph-mon[86649]: osdmap e663: 8 total, 8 up, 8 in 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3189: done (1 left) 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3190: read oid 5 snap -1 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3190: expect (ObjNum 991 snap 288 seq_num 991) 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3191: copy_from oid 26 from oid 30 current snap is 291 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3188: finishing write tid 1 to smithi03385152-47 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3188: finishing write tid 2 to smithi03385152-47 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3188: finishing write tid 3 to smithi03385152-47 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3188: finishing write tid 4 to smithi03385152-47 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3188: finishing write tid 5 to smithi03385152-47 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:3188: finishing write tid 6 to smithi03385152-47 2024-09-21T10:01:36.625 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1508 (ObjNum 995 snap 291 seq_num 995) dirty exists 2024-09-21T10:01:36.626 INFO:tasks.rados.rados.0.smithi033.stdout:3188: left oid 47 (ObjNum 995 snap 291 seq_num 995) 2024-09-21T10:01:36.626 INFO:tasks.rados.rados.0.smithi033.stdout:3188: done (2 left) 2024-09-21T10:01:36.626 INFO:tasks.rados.rados.0.smithi033.stdout:3192: delete oid 23 current snap is 291 2024-09-21T10:01:36.630 INFO:tasks.rados.rados.0.smithi033.stdout:3192: done (2 left) 2024-09-21T10:01:36.630 INFO:tasks.rados.rados.0.smithi033.stdout:3193: snap_create 2024-09-21T10:01:36.643 INFO:tasks.rados.rados.0.smithi033.stdout:3190: expect (ObjNum 991 snap 288 seq_num 991) 2024-09-21T10:01:36.748 INFO:tasks.rados.rados.0.smithi033.stdout:3191: finishing copy_from racing read to smithi03385152-26 2024-09-21T10:01:36.748 INFO:tasks.rados.rados.0.smithi033.stdout:3191: finishing copy_from to smithi03385152-26 2024-09-21T10:01:36.748 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 2347 (ObjNum 982 snap 286 seq_num 982) dirty exists 2024-09-21T10:01:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:36 smithi033 ceph-mon[98546]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T10:01:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:36 smithi033 ceph-mon[96568]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T10:01:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:36 smithi105 ceph-mon[86649]: osdmap e664: 8 total, 8 up, 8 in 2024-09-21T10:01:37.625 INFO:tasks.rados.rados.0.smithi033.stdout:3190: done (2 left) 2024-09-21T10:01:37.625 INFO:tasks.rados.rados.0.smithi033.stdout:3191: done (1 left) 2024-09-21T10:01:37.625 INFO:tasks.rados.rados.0.smithi033.stdout:3193: done (0 left) 2024-09-21T10:01:37.625 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 292 2024-09-21T10:01:37.625 INFO:tasks.rados.rados.0.smithi033.stdout:3194: seq_num 997 ranges {3694592=704512,4399104=794624,5193728=794624,5988352=16384} 2024-09-21T10:01:37.649 INFO:tasks.rados.rados.0.smithi033.stdout:3194: writing smithi03385152-29 from 3694592 to 4399104 tid 1 2024-09-21T10:01:37.653 INFO:tasks.rados.rados.0.smithi033.stdout:3194: writing smithi03385152-29 from 4399104 to 5193728 tid 2 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3194: writing smithi03385152-29 from 5193728 to 5988352 tid 3 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3194: writing smithi03385152-29 from 5988352 to 6004736 tid 4 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3195: read oid 15 snap -1 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3195: expect (ObjNum 962 snap 281 seq_num 962) 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3196: read oid 26 snap -1 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3196: expect (ObjNum 982 snap 286 seq_num 982) 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3197: rollback oid 6 current snap is 292 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3194: finishing write tid 1 to smithi03385152-29 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 291 2024-09-21T10:01:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3198: rollback oid 17 current snap is 292 2024-09-21T10:01:37.660 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 291 2024-09-21T10:01:37.660 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 292 2024-09-21T10:01:37.660 INFO:tasks.rados.rados.0.smithi033.stdout:3199: seq_num 998 ranges {2277376=720896,2998272=679936,3678208=598016,4276224=327680} 2024-09-21T10:01:37.676 INFO:tasks.rados.rados.0.smithi033.stdout:3199: writing smithi03385152-47 from 2277376 to 2998272 tid 1 2024-09-21T10:01:37.681 INFO:tasks.rados.rados.0.smithi033.stdout:3199: writing smithi03385152-47 from 2998272 to 3678208 tid 2 2024-09-21T10:01:37.685 INFO:tasks.rados.rados.0.smithi033.stdout:3199: writing smithi03385152-47 from 3678208 to 4276224 tid 3 2024-09-21T10:01:37.687 INFO:tasks.rados.rados.0.smithi033.stdout:3199: writing smithi03385152-47 from 4276224 to 4603904 tid 4 2024-09-21T10:01:37.687 INFO:tasks.rados.rados.0.smithi033.stdout:3200: rmattr oid 44 current snap is 292 2024-09-21T10:01:37.687 INFO:tasks.rados.rados.0.smithi033.stdout:3200: done (6 left) 2024-09-21T10:01:37.687 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 292 2024-09-21T10:01:37.687 INFO:tasks.rados.rados.0.smithi033.stdout:3194: finishing write tid 2 to smithi03385152-29 2024-09-21T10:01:37.687 INFO:tasks.rados.rados.0.smithi033.stdout:3194: finishing write tid 3 to smithi03385152-29 2024-09-21T10:01:37.687 INFO:tasks.rados.rados.0.smithi033.stdout:3194: finishing write tid 4 to smithi03385152-29 2024-09-21T10:01:37.688 INFO:tasks.rados.rados.0.smithi033.stdout:3201: seq_num 999 ranges {57344=737280,794624=565248,1359872=761856,2121728=278528} 2024-09-21T10:01:37.691 INFO:tasks.rados.rados.0.smithi033.stdout:3201: writing smithi03385152-41 from 57344 to 794624 tid 1 2024-09-21T10:01:37.695 INFO:tasks.rados.rados.0.smithi033.stdout:3201: writing smithi03385152-41 from 794624 to 1359872 tid 2 2024-09-21T10:01:37.699 INFO:tasks.rados.rados.0.smithi033.stdout:3201: writing smithi03385152-41 from 1359872 to 2121728 tid 3 2024-09-21T10:01:37.701 INFO:tasks.rados.rados.0.smithi033.stdout:3201: writing smithi03385152-41 from 2121728 to 2400256 tid 4 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3202: setattr oid 46 current snap is 292 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3194: finishing write tid 5 to smithi03385152-29 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3194: finishing write tid 6 to smithi03385152-29 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2366 (ObjNum 997 snap 292 seq_num 997) dirty exists 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3194: left oid 29 (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3197: finishing rollback tid 0 to smithi03385152-6 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3197: finishing rollback tid 1 to smithi03385152-6 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3197: finishing rollback tid 2 to smithi03385152-6 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 1627 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3199: finishing write tid 1 to smithi03385152-47 2024-09-21T10:01:37.702 INFO:tasks.rados.rados.0.smithi033.stdout:3196: expect (ObjNum 982 snap 286 seq_num 982) 2024-09-21T10:01:37.759 INFO:tasks.rados.rados.0.smithi033.stdout:3198: finishing rollback tid 0 to smithi03385152-17 2024-09-21T10:01:37.759 INFO:tasks.rados.rados.0.smithi033.stdout:3198: finishing rollback tid 1 to smithi03385152-17 2024-09-21T10:01:37.759 INFO:tasks.rados.rados.0.smithi033.stdout:3198: finishing rollback tid 2 to smithi03385152-17 2024-09-21T10:01:37.759 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2350 (ObjNum 947 snap 279 seq_num 947) dirty exists 2024-09-21T10:01:37.759 INFO:tasks.rados.rados.0.smithi033.stdout:3199: finishing write tid 2 to smithi03385152-47 2024-09-21T10:01:37.759 INFO:tasks.rados.rados.0.smithi033.stdout:3199: finishing write tid 3 to smithi03385152-47 2024-09-21T10:01:37.759 INFO:tasks.rados.rados.0.smithi033.stdout:3195: expect (ObjNum 962 snap 281 seq_num 962) 2024-09-21T10:01:37.971 INFO:tasks.rados.rados.0.smithi033.stdout:3199: finishing write tid 4 to smithi03385152-47 2024-09-21T10:01:37.971 INFO:tasks.rados.rados.0.smithi033.stdout:3199: finishing write tid 5 to smithi03385152-47 2024-09-21T10:01:37.971 INFO:tasks.rados.rados.0.smithi033.stdout:3199: finishing write tid 6 to smithi03385152-47 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1513 (ObjNum 998 snap 292 seq_num 998) dirty exists 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3199: left oid 47 (ObjNum 998 snap 292 seq_num 998) 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3201: finishing write tid 1 to smithi03385152-41 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3201: finishing write tid 2 to smithi03385152-41 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3194: done (7 left) 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3195: done (6 left) 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3196: done (5 left) 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3197: done (4 left) 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3198: done (3 left) 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3199: done (2 left) 2024-09-21T10:01:37.972 INFO:tasks.rados.rados.0.smithi033.stdout:3203: snap_remove snap 291 2024-09-21T10:01:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:37 smithi033 ceph-mon[98546]: pgmap v628: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 8 op/s 2024-09-21T10:01:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:37 smithi033 ceph-mon[98546]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T10:01:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:37 smithi033 ceph-mon[96568]: pgmap v628: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 8 op/s 2024-09-21T10:01:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:37 smithi033 ceph-mon[96568]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T10:01:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:37 smithi105 ceph-mon[86649]: pgmap v628: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.4 MiB/s wr, 8 op/s 2024-09-21T10:01:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:37 smithi105 ceph-mon[86649]: osdmap e665: 8 total, 8 up, 8 in 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3203: done (2 left) 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3204: copy_from oid 45 from oid 29 current snap is 292 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3201: finishing write tid 3 to smithi03385152-41 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3201: finishing write tid 4 to smithi03385152-41 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3201: finishing write tid 5 to smithi03385152-41 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3201: finishing write tid 6 to smithi03385152-41 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 1815 (ObjNum 999 snap 292 seq_num 999) dirty exists 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3201: left oid 41 (ObjNum 999 snap 292 seq_num 999) 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1876 (ObjNum 863 snap 253 seq_num 863) dirty exists 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3201: done (2 left) 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3202: done (1 left) 2024-09-21T10:01:38.546 INFO:tasks.rados.rados.0.smithi033.stdout:3205: rollback oid 44 current snap is 292 2024-09-21T10:01:38.547 INFO:tasks.rados.rados.0.smithi033.stdout:3205: done (1 left) 2024-09-21T10:01:38.547 INFO:tasks.rados.rados.0.smithi033.stdout:3206: copy_from oid 36 from oid 34 current snap is 292 2024-09-21T10:01:38.547 INFO:tasks.rados.rados.0.smithi033.stdout:3207: copy_from oid 16 from oid 49 current snap is 292 2024-09-21T10:01:38.547 INFO:tasks.rados.rados.0.smithi033.stdout:3208: rmattr oid 6 current snap is 292 2024-09-21T10:01:38.547 INFO:tasks.rados.rados.0.smithi033.stdout:3208: done (3 left) 2024-09-21T10:01:38.547 INFO:tasks.rados.rados.0.smithi033.stdout:3209: delete oid 39 current snap is 292 2024-09-21T10:01:38.550 INFO:tasks.rados.rados.0.smithi033.stdout:3206: finishing copy_from to smithi03385152-36 2024-09-21T10:01:38.551 INFO:tasks.rados.rados.0.smithi033.stdout:3206: got expected ENOENT (src dne) 2024-09-21T10:01:38.555 INFO:tasks.rados.rados.0.smithi033.stdout:3209: done (3 left) 2024-09-21T10:01:38.555 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 292 2024-09-21T10:01:38.555 INFO:tasks.rados.rados.0.smithi033.stdout:3210: seq_num 1001 ranges {3923968=770048,4694016=729088,5423104=770048,6193152=106496} 2024-09-21T10:01:38.581 INFO:tasks.rados.rados.0.smithi033.stdout:3210: writing smithi03385152-8 from 3923968 to 4694016 tid 1 2024-09-21T10:01:38.585 INFO:tasks.rados.rados.0.smithi033.stdout:3210: writing smithi03385152-8 from 4694016 to 5423104 tid 2 2024-09-21T10:01:38.590 INFO:tasks.rados.rados.0.smithi033.stdout:3210: writing smithi03385152-8 from 5423104 to 6193152 tid 3 2024-09-21T10:01:38.591 INFO:tasks.rados.rados.0.smithi033.stdout:3210: writing smithi03385152-8 from 6193152 to 6299648 tid 4 2024-09-21T10:01:38.591 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 292 2024-09-21T10:01:38.591 INFO:tasks.rados.rados.0.smithi033.stdout:3206: finishing copy_from racing read to smithi03385152-36 2024-09-21T10:01:38.591 INFO:tasks.rados.rados.0.smithi033.stdout:3211: seq_num 1002 ranges {4210688=786432,4997120=614400,5611520=581632,6193152=409600} 2024-09-21T10:01:38.618 INFO:tasks.rados.rados.0.smithi033.stdout:3211: writing smithi03385152-38 from 4210688 to 4997120 tid 1 2024-09-21T10:01:38.622 INFO:tasks.rados.rados.0.smithi033.stdout:3211: writing smithi03385152-38 from 4997120 to 5611520 tid 2 2024-09-21T10:01:38.626 INFO:tasks.rados.rados.0.smithi033.stdout:3211: writing smithi03385152-38 from 5611520 to 6193152 tid 3 2024-09-21T10:01:38.628 INFO:tasks.rados.rados.0.smithi033.stdout:3211: writing smithi03385152-38 from 6193152 to 6602752 tid 4 2024-09-21T10:01:38.628 INFO:tasks.rados.rados.0.smithi033.stdout:3206: done (4 left) 2024-09-21T10:01:38.628 INFO:tasks.rados.rados.0.smithi033.stdout:3212: setattr oid 24 current snap is 292 2024-09-21T10:01:38.628 INFO:tasks.rados.rados.0.smithi033.stdout:3207: finishing copy_from racing read to smithi03385152-16 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3207: finishing copy_from to smithi03385152-16 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 1643 (ObjNum 987 snap 288 seq_num 987) dirty exists 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3204: finishing copy_from racing read to smithi03385152-45 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3204: finishing copy_from to smithi03385152-45 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2366 (ObjNum 997 snap 292 seq_num 997) dirty exists 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3210: finishing write tid 1 to smithi03385152-8 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3210: finishing write tid 2 to smithi03385152-8 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3210: finishing write tid 3 to smithi03385152-8 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3210: finishing write tid 4 to smithi03385152-8 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3210: finishing write tid 5 to smithi03385152-8 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3210: finishing write tid 6 to smithi03385152-8 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 1518 (ObjNum 1001 snap 292 seq_num 1001) dirty exists 2024-09-21T10:01:38.629 INFO:tasks.rados.rados.0.smithi033.stdout:3210: left oid 8 (ObjNum 1001 snap 292 seq_num 1001) 2024-09-21T10:01:38.630 INFO:tasks.rados.rados.0.smithi033.stdout:3211: finishing write tid 1 to smithi03385152-38 2024-09-21T10:01:38.630 INFO:tasks.rados.rados.0.smithi033.stdout:3211: finishing write tid 2 to smithi03385152-38 2024-09-21T10:01:38.630 INFO:tasks.rados.rados.0.smithi033.stdout:3204: done (4 left) 2024-09-21T10:01:38.630 INFO:tasks.rados.rados.0.smithi033.stdout:3207: done (3 left) 2024-09-21T10:01:38.631 INFO:tasks.rados.rados.0.smithi033.stdout:3210: done (2 left) 2024-09-21T10:01:38.631 INFO:tasks.rados.rados.0.smithi033.stdout:3213: copy_from oid 15 from oid 34 current snap is 292 2024-09-21T10:01:38.631 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 292 2024-09-21T10:01:38.631 INFO:tasks.rados.rados.0.smithi033.stdout:3214: seq_num 1003 ranges {0=802816,802816=499712,1302528=737280,2039808=368640} 2024-09-21T10:01:38.635 INFO:tasks.rados.rados.0.smithi033.stdout:3214: writing smithi03385152-36 from 0 to 802816 tid 1 2024-09-21T10:01:38.639 INFO:tasks.rados.rados.0.smithi033.stdout:3214: writing smithi03385152-36 from 802816 to 1302528 tid 2 2024-09-21T10:01:38.643 INFO:tasks.rados.rados.0.smithi033.stdout:3214: writing smithi03385152-36 from 1302528 to 2039808 tid 3 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3214: writing smithi03385152-36 from 2039808 to 2408448 tid 4 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3215: copy_from oid 40 from oid 5 current snap is 292 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3213: finishing copy_from to smithi03385152-15 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3213: got expected ENOENT (src dne) 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3213: finishing copy_from racing read to smithi03385152-15 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3211: finishing write tid 3 to smithi03385152-38 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3211: finishing write tid 4 to smithi03385152-38 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3211: finishing write tid 5 to smithi03385152-38 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3213: done (4 left) 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:3216: delete oid 6 current snap is 292 2024-09-21T10:01:38.646 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 2264 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:01:38.647 INFO:tasks.rados.rados.0.smithi033.stdout:3211: finishing write tid 6 to smithi03385152-38 2024-09-21T10:01:38.647 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 1820 (ObjNum 1002 snap 292 seq_num 1002) dirty exists 2024-09-21T10:01:38.647 INFO:tasks.rados.rados.0.smithi033.stdout:3211: left oid 38 (ObjNum 1002 snap 292 seq_num 1002) 2024-09-21T10:01:38.647 INFO:tasks.rados.rados.0.smithi033.stdout:3214: finishing write tid 1 to smithi03385152-36 2024-09-21T10:01:38.647 INFO:tasks.rados.rados.0.smithi033.stdout:3214: finishing write tid 2 to smithi03385152-36 2024-09-21T10:01:38.648 INFO:tasks.rados.rados.0.smithi033.stdout:3211: done (4 left) 2024-09-21T10:01:38.648 INFO:tasks.rados.rados.0.smithi033.stdout:3212: done (3 left) 2024-09-21T10:01:38.648 INFO:tasks.rados.rados.0.smithi033.stdout:3216: done (2 left) 2024-09-21T10:01:38.648 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 292 2024-09-21T10:01:38.648 INFO:tasks.rados.rados.0.smithi033.stdout:3217: seq_num 1005 ranges {57344=32768} 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3217: writing smithi03385152-46 from 57344 to 90112 tid 1 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3218: read oid 28 snap -1 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3218: expect (ObjNum 914 snap 265 seq_num 914) 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3219: rollback oid 8 current snap is 292 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3219: done (4 left) 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3220: snap_create 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3214: finishing write tid 3 to smithi03385152-36 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3214: finishing write tid 4 to smithi03385152-36 2024-09-21T10:01:38.649 INFO:tasks.rados.rados.0.smithi033.stdout:3214: finishing write tid 5 to smithi03385152-36 2024-09-21T10:01:38.652 INFO:tasks.rados.rados.0.smithi033.stdout:3217: finishing write tid 1 to smithi03385152-46 2024-09-21T10:01:38.652 INFO:tasks.rados.rados.0.smithi033.stdout:3214: finishing write tid 6 to smithi03385152-36 2024-09-21T10:01:38.652 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2374 (ObjNum 1003 snap 292 seq_num 1003) dirty exists 2024-09-21T10:01:38.652 INFO:tasks.rados.rados.0.smithi033.stdout:3214: left oid 36 (ObjNum 1003 snap 292 seq_num 1003) 2024-09-21T10:01:38.653 INFO:tasks.rados.rados.0.smithi033.stdout:3217: finishing write tid 2 to smithi03385152-46 2024-09-21T10:01:38.653 INFO:tasks.rados.rados.0.smithi033.stdout:3218: expect (ObjNum 914 snap 265 seq_num 914) 2024-09-21T10:01:38.682 INFO:tasks.rados.rados.0.smithi033.stdout:3217: finishing write tid 3 to smithi03385152-46 2024-09-21T10:01:38.682 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2368 (ObjNum 1005 snap 292 seq_num 1005) dirty exists 2024-09-21T10:01:38.682 INFO:tasks.rados.rados.0.smithi033.stdout:3217: left oid 46 (ObjNum 1005 snap 292 seq_num 1005) 2024-09-21T10:01:38.682 INFO:tasks.rados.rados.0.smithi033.stdout:3215: finishing copy_from racing read to smithi03385152-40 2024-09-21T10:01:38.682 INFO:tasks.rados.rados.0.smithi033.stdout:3215: finishing copy_from to smithi03385152-40 2024-09-21T10:01:38.682 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1065 (ObjNum 991 snap 288 seq_num 991) dirty exists 2024-09-21T10:01:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:38 smithi033 ceph-mon[98546]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T10:01:38.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:38 smithi033 ceph-mon[98546]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T10:01:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:38 smithi033 ceph-mon[96568]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T10:01:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:38 smithi033 ceph-mon[96568]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T10:01:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:38 smithi105 ceph-mon[86649]: osdmap e666: 8 total, 8 up, 8 in 2024-09-21T10:01:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:38 smithi105 ceph-mon[86649]: osdmap e667: 8 total, 8 up, 8 in 2024-09-21T10:01:39.030 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:01:38 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:01:38.563+0000 7f42b8457640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 179 after trimming was started 2024-09-21T10:01:39.643 INFO:tasks.rados.rados.0.smithi033.stdout:3214: done (4 left) 2024-09-21T10:01:39.643 INFO:tasks.rados.rados.0.smithi033.stdout:3215: done (3 left) 2024-09-21T10:01:39.643 INFO:tasks.rados.rados.0.smithi033.stdout:3217: done (2 left) 2024-09-21T10:01:39.643 INFO:tasks.rados.rados.0.smithi033.stdout:3218: done (1 left) 2024-09-21T10:01:39.643 INFO:tasks.rados.rados.0.smithi033.stdout:3220: done (0 left) 2024-09-21T10:01:39.643 INFO:tasks.rados.rados.0.smithi033.stdout:3221: rmattr oid 4 current snap is 293 2024-09-21T10:01:39.644 INFO:tasks.rados.rados.0.smithi033.stdout:3221: done (0 left) 2024-09-21T10:01:39.644 INFO:tasks.rados.rados.0.smithi033.stdout:3222: snap_create 2024-09-21T10:01:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:39 smithi033 ceph-mon[98546]: pgmap v631: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3 op/s 2024-09-21T10:01:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:39 smithi033 ceph-mon[96568]: pgmap v631: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3 op/s 2024-09-21T10:01:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:39 smithi105 ceph-mon[86649]: pgmap v631: 121 pgs: 121 active+clean; 128 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 3 op/s 2024-09-21T10:01:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:01:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:01:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:40.652 INFO:tasks.rados.rados.0.smithi033.stdout:3222: done (0 left) 2024-09-21T10:01:40.652 INFO:tasks.rados.rados.0.smithi033.stdout:3223: snap_create 2024-09-21T10:01:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:40 smithi033 ceph-mon[98546]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T10:01:40.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:40 smithi033 ceph-mon[96568]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T10:01:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:40 smithi105 ceph-mon[86649]: osdmap e668: 8 total, 8 up, 8 in 2024-09-21T10:01:41.655 INFO:tasks.rados.rados.0.smithi033.stdout:3223: done (0 left) 2024-09-21T10:01:41.655 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 295 2024-09-21T10:01:41.655 INFO:tasks.rados.rados.0.smithi033.stdout:3224: seq_num 1006 ranges {4784128=49152} 2024-09-21T10:01:41.682 INFO:tasks.rados.rados.0.smithi033.stdout:3224: writing smithi03385152-15 from 4784128 to 4833280 tid 1 2024-09-21T10:01:41.682 INFO:tasks.rados.rados.0.smithi033.stdout:3225: rollback oid 3 current snap is 295 2024-09-21T10:01:41.682 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 3 to 293 2024-09-21T10:01:41.682 INFO:tasks.rados.rados.0.smithi033.stdout:3226: read oid 50 snap 292 2024-09-21T10:01:41.682 INFO:tasks.rados.rados.0.smithi033.stdout:3226: expect (ObjNum 965 snap 283 seq_num 965) 2024-09-21T10:01:41.682 INFO:tasks.rados.rados.0.smithi033.stdout:3227: snap_create 2024-09-21T10:01:41.686 INFO:tasks.rados.rados.0.smithi033.stdout:3224: finishing write tid 1 to smithi03385152-15 2024-09-21T10:01:41.686 INFO:tasks.rados.rados.0.smithi033.stdout:3224: finishing write tid 2 to smithi03385152-15 2024-09-21T10:01:41.687 INFO:tasks.rados.rados.0.smithi033.stdout:3224: finishing write tid 3 to smithi03385152-15 2024-09-21T10:01:41.687 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 1520 (ObjNum 1006 snap 295 seq_num 1006) dirty exists 2024-09-21T10:01:41.688 INFO:tasks.rados.rados.0.smithi033.stdout:3224: left oid 15 (ObjNum 1006 snap 295 seq_num 1006) 2024-09-21T10:01:41.690 INFO:tasks.rados.rados.0.smithi033.stdout:3225: finishing rollback tid 0 to smithi03385152-3 2024-09-21T10:01:41.691 INFO:tasks.rados.rados.0.smithi033.stdout:3225: finishing rollback tid 1 to smithi03385152-3 2024-09-21T10:01:41.691 INFO:tasks.rados.rados.0.smithi033.stdout:3225: finishing rollback tid 2 to smithi03385152-3 2024-09-21T10:01:41.691 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2371 (ObjNum 964 snap 283 seq_num 964) dirty exists 2024-09-21T10:01:41.694 INFO:tasks.rados.rados.0.smithi033.stdout:3226: expect (ObjNum 965 snap 283 seq_num 965) 2024-09-21T10:01:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:01:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:01:41] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-09-21T10:01:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:41 smithi033 ceph-mon[98546]: pgmap v634: 121 pgs: 121 active+clean; 128 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T10:01:41.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:41 smithi033 ceph-mon[98546]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T10:01:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:41 smithi033 ceph-mon[96568]: pgmap v634: 121 pgs: 121 active+clean; 128 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T10:01:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:41 smithi033 ceph-mon[96568]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T10:01:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:41 smithi105 ceph-mon[86649]: pgmap v634: 121 pgs: 121 active+clean; 128 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 2.2 MiB/s wr, 10 op/s 2024-09-21T10:01:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:41 smithi105 ceph-mon[86649]: osdmap e669: 8 total, 8 up, 8 in 2024-09-21T10:01:42.673 INFO:tasks.rados.rados.0.smithi033.stdout:3224: done (3 left) 2024-09-21T10:01:42.673 INFO:tasks.rados.rados.0.smithi033.stdout:3225: done (2 left) 2024-09-21T10:01:42.673 INFO:tasks.rados.rados.0.smithi033.stdout:3226: done (1 left) 2024-09-21T10:01:42.673 INFO:tasks.rados.rados.0.smithi033.stdout:3227: done (0 left) 2024-09-21T10:01:42.673 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 296 2024-09-21T10:01:42.673 INFO:tasks.rados.rados.0.smithi033.stdout:3228: seq_num 1007 ranges {4300800=65536} 2024-09-21T10:01:42.697 INFO:tasks.rados.rados.0.smithi033.stdout:3228: writing smithi03385152-7 from 4300800 to 4366336 tid 1 2024-09-21T10:01:42.697 INFO:tasks.rados.rados.0.smithi033.stdout:3229: rollback oid 44 current snap is 296 2024-09-21T10:01:42.697 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 44 to 292 2024-09-21T10:01:42.697 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 296 2024-09-21T10:01:42.697 INFO:tasks.rados.rados.0.smithi033.stdout:3230: seq_num 1008 ranges {2441216=90112} 2024-09-21T10:01:42.711 INFO:tasks.rados.rados.0.smithi033.stdout:3230: writing smithi03385152-42 from 2441216 to 2531328 tid 1 2024-09-21T10:01:42.711 INFO:tasks.rados.rados.0.smithi033.stdout:3231: rollback oid 9 current snap is 296 2024-09-21T10:01:42.711 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 9 to 292 2024-09-21T10:01:42.711 INFO:tasks.rados.rados.0.smithi033.stdout:3229: finishing rollback tid 1 to smithi03385152-44 2024-09-21T10:01:42.711 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 0 (ObjNum 33 snap 0 seq_num 72668536) dirty dne 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3228: finishing write tid 1 to smithi03385152-7 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3228: finishing write tid 2 to smithi03385152-7 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3228: finishing write tid 3 to smithi03385152-7 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2377 (ObjNum 1007 snap 296 seq_num 1007) dirty exists 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3228: left oid 7 (ObjNum 1007 snap 296 seq_num 1007) 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3228: done (3 left) 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3229: done (2 left) 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3232: read oid 14 snap -1 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3232: expect (ObjNum 952 snap 279 seq_num 952) 2024-09-21T10:01:42.712 INFO:tasks.rados.rados.0.smithi033.stdout:3233: snap_create 2024-09-21T10:01:42.717 INFO:tasks.rados.rados.0.smithi033.stdout:3232: expect (ObjNum 952 snap 279 seq_num 952) 2024-09-21T10:01:42.761 INFO:tasks.rados.rados.0.smithi033.stdout:3230: finishing write tid 1 to smithi03385152-42 2024-09-21T10:01:42.761 INFO:tasks.rados.rados.0.smithi033.stdout:3230: finishing write tid 2 to smithi03385152-42 2024-09-21T10:01:42.761 INFO:tasks.rados.rados.0.smithi033.stdout:3231: finishing rollback tid 0 to smithi03385152-9 2024-09-21T10:01:42.761 INFO:tasks.rados.rados.0.smithi033.stdout:3231: finishing rollback tid 1 to smithi03385152-9 2024-09-21T10:01:42.761 INFO:tasks.rados.rados.0.smithi033.stdout:3230: finishing write tid 3 to smithi03385152-42 2024-09-21T10:01:42.762 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 1822 (ObjNum 1008 snap 296 seq_num 1008) dirty exists 2024-09-21T10:01:42.762 INFO:tasks.rados.rados.0.smithi033.stdout:3230: left oid 42 (ObjNum 1008 snap 296 seq_num 1008) 2024-09-21T10:01:42.762 INFO:tasks.rados.rados.0.smithi033.stdout:3231: finishing rollback tid 2 to smithi03385152-9 2024-09-21T10:01:42.762 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 1825 (ObjNum 965 snap 283 seq_num 965) dirty exists 2024-09-21T10:01:42.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:42 smithi033 ceph-mon[98546]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T10:01:42.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:42 smithi033 ceph-mon[96568]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T10:01:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:42 smithi105 ceph-mon[86649]: osdmap e670: 8 total, 8 up, 8 in 2024-09-21T10:01:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:3230: done (3 left) 2024-09-21T10:01:43.547 INFO:tasks.rados.rados.0.smithi033.stdout:3231: done (2 left) 2024-09-21T10:01:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:3232: done (1 left) 2024-09-21T10:01:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:3233: done (0 left) 2024-09-21T10:01:43.548 INFO:tasks.rados.rados.0.smithi033.stdout:3234: snap_remove snap 293 2024-09-21T10:01:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:43 smithi033 ceph-mon[98546]: pgmap v637: 121 pgs: 121 active+clean; 137 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.8 MiB/s wr, 23 op/s 2024-09-21T10:01:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:43 smithi033 ceph-mon[98546]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T10:01:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:43 smithi033 ceph-mon[98546]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T10:01:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:43 smithi033 ceph-mon[96568]: pgmap v637: 121 pgs: 121 active+clean; 137 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.8 MiB/s wr, 23 op/s 2024-09-21T10:01:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:43 smithi033 ceph-mon[96568]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T10:01:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:43 smithi033 ceph-mon[96568]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T10:01:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:43 smithi105 ceph-mon[86649]: pgmap v637: 121 pgs: 121 active+clean; 137 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.0 MiB/s rd, 5.8 MiB/s wr, 23 op/s 2024-09-21T10:01:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:43 smithi105 ceph-mon[86649]: osdmap e671: 8 total, 8 up, 8 in 2024-09-21T10:01:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:43 smithi105 ceph-mon[86649]: osdmap e672: 8 total, 8 up, 8 in 2024-09-21T10:01:44.549 INFO:tasks.rados.rados.0.smithi033.stdout:3234: done (0 left) 2024-09-21T10:01:44.549 INFO:tasks.rados.rados.0.smithi033.stdout:3235: rmattr oid 38 current snap is 297 2024-09-21T10:01:44.550 INFO:tasks.rados.rados.0.smithi033.stdout:3235: done (0 left) 2024-09-21T10:01:44.550 INFO:tasks.rados.rados.0.smithi033.stdout:3236: read oid 15 snap -1 2024-09-21T10:01:44.550 INFO:tasks.rados.rados.0.smithi033.stdout:3236: expect (ObjNum 1006 snap 295 seq_num 1006) 2024-09-21T10:01:44.550 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 297 2024-09-21T10:01:44.550 INFO:tasks.rados.rados.0.smithi033.stdout:3237: seq_num 1009 ranges {0=106496} 2024-09-21T10:01:44.551 INFO:tasks.rados.rados.0.smithi033.stdout:3237: writing smithi03385152-27 from 0 to 106496 tid 1 2024-09-21T10:01:44.551 INFO:tasks.rados.rados.0.smithi033.stdout:3238: read oid 24 snap -1 2024-09-21T10:01:44.551 INFO:tasks.rados.rados.0.smithi033.stdout:3238: expect (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:01:44.551 INFO:tasks.rados.rados.0.smithi033.stdout:3239: snap_create 2024-09-21T10:01:44.559 INFO:tasks.rados.rados.0.smithi033.stdout:3237: finishing write tid 1 to smithi03385152-27 2024-09-21T10:01:44.560 INFO:tasks.rados.rados.0.smithi033.stdout:3237: finishing write tid 2 to smithi03385152-27 2024-09-21T10:01:44.565 INFO:tasks.rados.rados.0.smithi033.stdout:3237: finishing write tid 3 to smithi03385152-27 2024-09-21T10:01:44.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1630 (ObjNum 1009 snap 297 seq_num 1009) dirty exists 2024-09-21T10:01:44.565 INFO:tasks.rados.rados.0.smithi033.stdout:3237: left oid 27 (ObjNum 1009 snap 297 seq_num 1009) 2024-09-21T10:01:44.566 INFO:tasks.rados.rados.0.smithi033.stdout:3238: expect (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:01:44.624 INFO:tasks.rados.rados.0.smithi033.stdout:3236: expect (ObjNum 1006 snap 295 seq_num 1006) 2024-09-21T10:01:45.030 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:01:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:01:44.564+0000 7ff76a7a5640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 17b after trimming was started 2024-09-21T10:01:45.551 INFO:tasks.rados.rados.0.smithi033.stdout:3236: done (3 left) 2024-09-21T10:01:45.551 INFO:tasks.rados.rados.0.smithi033.stdout:3237: done (2 left) 2024-09-21T10:01:45.551 INFO:tasks.rados.rados.0.smithi033.stdout:3238: done (1 left) 2024-09-21T10:01:45.551 INFO:tasks.rados.rados.0.smithi033.stdout:3239: done (0 left) 2024-09-21T10:01:45.551 INFO:tasks.rados.rados.0.smithi033.stdout:3240: read oid 8 snap -1 2024-09-21T10:01:45.551 INFO:tasks.rados.rados.0.smithi033.stdout:3240: expect (ObjNum 1001 snap 292 seq_num 1001) 2024-09-21T10:01:45.551 INFO:tasks.rados.rados.0.smithi033.stdout:3241: delete oid 2 current snap is 298 2024-09-21T10:01:45.554 INFO:tasks.rados.rados.0.smithi033.stdout:3241: done (1 left) 2024-09-21T10:01:45.555 INFO:tasks.rados.rados.0.smithi033.stdout:3242: snap_remove snap 297 2024-09-21T10:01:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:45 smithi033 ceph-mon[98546]: pgmap v640: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.0 MiB/s wr, 22 op/s 2024-09-21T10:01:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:45 smithi033 ceph-mon[98546]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T10:01:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:45 smithi033 ceph-mon[96568]: pgmap v640: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.0 MiB/s wr, 22 op/s 2024-09-21T10:01:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:45 smithi033 ceph-mon[96568]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T10:01:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:45 smithi105 ceph-mon[86649]: pgmap v640: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.0 MiB/s wr, 22 op/s 2024-09-21T10:01:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:45 smithi105 ceph-mon[86649]: osdmap e673: 8 total, 8 up, 8 in 2024-09-21T10:01:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:3242: done (1 left) 2024-09-21T10:01:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 298 2024-09-21T10:01:46.569 INFO:tasks.rados.rados.0.smithi033.stdout:3243: seq_num 1011 ranges {4603904=139264} 2024-09-21T10:01:46.595 INFO:tasks.rados.rados.0.smithi033.stdout:3243: writing smithi03385152-47 from 4603904 to 4743168 tid 1 2024-09-21T10:01:46.595 INFO:tasks.rados.rados.0.smithi033.stdout:3244: snap_remove snap 294 2024-09-21T10:01:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:46 smithi033 ceph-mon[98546]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T10:01:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:46 smithi033 ceph-mon[96568]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T10:01:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:46 smithi105 ceph-mon[86649]: osdmap e674: 8 total, 8 up, 8 in 2024-09-21T10:01:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:01:47.570 INFO:tasks.rados.rados.0.smithi033.stdout:3244: done (2 left) 2024-09-21T10:01:47.570 INFO:tasks.rados.rados.0.smithi033.stdout:3245: snap_create 2024-09-21T10:01:47.570 INFO:tasks.rados.rados.0.smithi033.stdout:3240: expect (ObjNum 1001 snap 292 seq_num 1001) 2024-09-21T10:01:47.849 INFO:tasks.rados.rados.0.smithi033.stdout:3243: finishing write tid 1 to smithi03385152-47 2024-09-21T10:01:47.849 INFO:tasks.rados.rados.0.smithi033.stdout:3243: finishing write tid 2 to smithi03385152-47 2024-09-21T10:01:47.849 INFO:tasks.rados.rados.0.smithi033.stdout:3243: finishing write tid 3 to smithi03385152-47 2024-09-21T10:01:47.849 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1515 (ObjNum 1011 snap 298 seq_num 1011) dirty exists 2024-09-21T10:01:47.849 INFO:tasks.rados.rados.0.smithi033.stdout:3243: left oid 47 (ObjNum 1011 snap 298 seq_num 1011) 2024-09-21T10:01:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:47 smithi033 ceph-mon[98546]: pgmap v643: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T10:01:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:47 smithi033 ceph-mon[98546]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T10:01:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:47 smithi033 ceph-mon[96568]: pgmap v643: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T10:01:47.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:47 smithi033 ceph-mon[96568]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T10:01:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:47 smithi105 ceph-mon[86649]: pgmap v643: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 1.4 MiB/s wr, 11 op/s 2024-09-21T10:01:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:47 smithi105 ceph-mon[86649]: osdmap e675: 8 total, 8 up, 8 in 2024-09-21T10:01:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:3240: done (2 left) 2024-09-21T10:01:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:3243: done (1 left) 2024-09-21T10:01:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:3245: done (0 left) 2024-09-21T10:01:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 299 2024-09-21T10:01:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:3246: seq_num 1012 ranges {0=155648} 2024-09-21T10:01:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:3246: writing smithi03385152-34 from 0 to 155648 tid 1 2024-09-21T10:01:48.549 INFO:tasks.rados.rados.0.smithi033.stdout:3247: setattr oid 41 current snap is 299 2024-09-21T10:01:48.550 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 299 2024-09-21T10:01:48.550 INFO:tasks.rados.rados.0.smithi033.stdout:3248: seq_num 1013 ranges {3293184=172032} 2024-09-21T10:01:48.570 INFO:tasks.rados.rados.0.smithi033.stdout:3248: writing smithi03385152-40 from 3293184 to 3465216 tid 1 2024-09-21T10:01:48.570 INFO:tasks.rados.rados.0.smithi033.stdout:3249: read oid 47 snap 298 2024-09-21T10:01:48.570 INFO:tasks.rados.rados.0.smithi033.stdout:3249: expect (ObjNum 1011 snap 298 seq_num 1011) 2024-09-21T10:01:48.570 INFO:tasks.rados.rados.0.smithi033.stdout:3250: read oid 12 snap -1 2024-09-21T10:01:48.570 INFO:tasks.rados.rados.0.smithi033.stdout:3250: expect deleted 2024-09-21T10:01:48.570 INFO:tasks.rados.rados.0.smithi033.stdout:3251: snap_remove snap 295 2024-09-21T10:01:48.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:48 smithi033 ceph-mon[98546]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T10:01:48.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:48 smithi033 ceph-mon[98546]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T10:01:48.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:48 smithi033 ceph-mon[96568]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T10:01:48.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:48 smithi033 ceph-mon[96568]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T10:01:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:48 smithi105 ceph-mon[86649]: osdmap e676: 8 total, 8 up, 8 in 2024-09-21T10:01:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:48 smithi105 ceph-mon[86649]: osdmap e677: 8 total, 8 up, 8 in 2024-09-21T10:01:49.551 INFO:tasks.rados.rados.0.smithi033.stdout:3251: done (5 left) 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3252: delete oid 42 current snap is 299 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 1826 (ObjNum 999 snap 292 seq_num 999) dirty exists 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3246: finishing write tid 1 to smithi03385152-34 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3246: finishing write tid 2 to smithi03385152-34 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3246: finishing write tid 3 to smithi03385152-34 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 2373 (ObjNum 1012 snap 299 seq_num 1012) dirty exists 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3246: left oid 34 (ObjNum 1012 snap 299 seq_num 1012) 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3248: finishing write tid 1 to smithi03385152-40 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3248: finishing write tid 2 to smithi03385152-40 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3248: finishing write tid 3 to smithi03385152-40 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1828 (ObjNum 1013 snap 299 seq_num 1013) dirty exists 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3248: left oid 40 (ObjNum 1013 snap 299 seq_num 1013) 2024-09-21T10:01:49.552 INFO:tasks.rados.rados.0.smithi033.stdout:3249: expect (ObjNum 1011 snap 298 seq_num 1011) 2024-09-21T10:01:49.795 INFO:tasks.rados.rados.0.smithi033.stdout:3246: done (5 left) 2024-09-21T10:01:49.795 INFO:tasks.rados.rados.0.smithi033.stdout:3247: done (4 left) 2024-09-21T10:01:49.796 INFO:tasks.rados.rados.0.smithi033.stdout:3248: done (3 left) 2024-09-21T10:01:49.796 INFO:tasks.rados.rados.0.smithi033.stdout:3249: done (2 left) 2024-09-21T10:01:49.796 INFO:tasks.rados.rados.0.smithi033.stdout:3250: done (1 left) 2024-09-21T10:01:49.796 INFO:tasks.rados.rados.0.smithi033.stdout:3252: done (0 left) 2024-09-21T10:01:49.796 INFO:tasks.rados.rados.0.smithi033.stdout:3253: snap_remove snap 298 2024-09-21T10:01:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:49 smithi033 ceph-mon[98546]: pgmap v646: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 28 KiB/s wr, 5 op/s 2024-09-21T10:01:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:49 smithi033 ceph-mon[98546]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T10:01:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:49 smithi033 ceph-mon[96568]: pgmap v646: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 28 KiB/s wr, 5 op/s 2024-09-21T10:01:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:49 smithi033 ceph-mon[96568]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T10:01:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:49 smithi105 ceph-mon[86649]: pgmap v646: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 28 KiB/s wr, 5 op/s 2024-09-21T10:01:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:49 smithi105 ceph-mon[86649]: osdmap e678: 8 total, 8 up, 8 in 2024-09-21T10:01:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:01:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:01:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:01:50.600 INFO:tasks.rados.rados.0.smithi033.stdout:3253: done (0 left) 2024-09-21T10:01:50.600 INFO:tasks.rados.rados.0.smithi033.stdout:3254: copy_from oid 33 from oid 15 current snap is 299 2024-09-21T10:01:50.600 INFO:tasks.rados.rados.0.smithi033.stdout:3255: copy_from oid 16 from oid 12 current snap is 299 2024-09-21T10:01:50.600 INFO:tasks.rados.rados.0.smithi033.stdout:3256: rmattr oid 8 current snap is 299 2024-09-21T10:01:50.601 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 299 2024-09-21T10:01:50.601 INFO:tasks.rados.rados.0.smithi033.stdout:3257: seq_num 1015 ranges {1753088=204800} 2024-09-21T10:01:50.611 INFO:tasks.rados.rados.0.smithi033.stdout:3257: writing smithi03385152-3 from 1753088 to 1957888 tid 1 2024-09-21T10:01:50.611 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 299 2024-09-21T10:01:50.611 INFO:tasks.rados.rados.0.smithi033.stdout:3258: seq_num 1016 ranges {2195456=221184} 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:3258: writing smithi03385152-19 from 2195456 to 2416640 tid 1 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 43 current snap is 299 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 1521 (ObjNum 1001 snap 292 seq_num 1001) dirty exists 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:3255: finishing copy_from to smithi03385152-16 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:3255: got expected ENOENT (src dne) 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:3255: finishing copy_from racing read to smithi03385152-16 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:3257: finishing write tid 1 to smithi03385152-3 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:3257: finishing write tid 2 to smithi03385152-3 2024-09-21T10:01:50.625 INFO:tasks.rados.rados.0.smithi033.stdout:3259: seq_num 1017 ranges {10870784=237568} 2024-09-21T10:01:50.686 INFO:tasks.rados.rados.0.smithi033.stdout:3259: writing smithi03385152-43 from 10870784 to 11108352 tid 1 2024-09-21T10:01:50.686 INFO:tasks.rados.rados.0.smithi033.stdout:3255: done (5 left) 2024-09-21T10:01:50.686 INFO:tasks.rados.rados.0.smithi033.stdout:3256: done (4 left) 2024-09-21T10:01:50.686 INFO:tasks.rados.rados.0.smithi033.stdout:3260: snap_create 2024-09-21T10:01:50.686 INFO:tasks.rados.rados.0.smithi033.stdout:3257: finishing write tid 3 to smithi03385152-3 2024-09-21T10:01:50.686 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2375 (ObjNum 1015 snap 299 seq_num 1015) dirty exists 2024-09-21T10:01:50.686 INFO:tasks.rados.rados.0.smithi033.stdout:3257: left oid 3 (ObjNum 1015 snap 299 seq_num 1015) 2024-09-21T10:01:50.687 INFO:tasks.rados.rados.0.smithi033.stdout:3254: finishing copy_from racing read to smithi03385152-33 2024-09-21T10:01:50.687 INFO:tasks.rados.rados.0.smithi033.stdout:3254: finishing copy_from to smithi03385152-33 2024-09-21T10:01:50.687 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 1520 (ObjNum 1006 snap 295 seq_num 1006) dirty exists 2024-09-21T10:01:50.687 INFO:tasks.rados.rados.0.smithi033.stdout:3258: finishing write tid 1 to smithi03385152-19 2024-09-21T10:01:50.687 INFO:tasks.rados.rados.0.smithi033.stdout:3258: finishing write tid 2 to smithi03385152-19 2024-09-21T10:01:50.687 INFO:tasks.rados.rados.0.smithi033.stdout:3258: finishing write tid 3 to smithi03385152-19 2024-09-21T10:01:50.687 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2266 (ObjNum 1016 snap 299 seq_num 1016) dirty exists 2024-09-21T10:01:50.687 INFO:tasks.rados.rados.0.smithi033.stdout:3258: left oid 19 (ObjNum 1016 snap 299 seq_num 1016) 2024-09-21T10:01:50.688 INFO:tasks.rados.rados.0.smithi033.stdout:3259: finishing write tid 1 to smithi03385152-43 2024-09-21T10:01:50.688 INFO:tasks.rados.rados.0.smithi033.stdout:3259: finishing write tid 2 to smithi03385152-43 2024-09-21T10:01:50.691 INFO:tasks.rados.rados.0.smithi033.stdout:3259: finishing write tid 3 to smithi03385152-43 2024-09-21T10:01:50.691 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1534 (ObjNum 1017 snap 299 seq_num 1017) dirty exists 2024-09-21T10:01:50.691 INFO:tasks.rados.rados.0.smithi033.stdout:3259: left oid 43 (ObjNum 1017 snap 299 seq_num 1017) 2024-09-21T10:01:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:01:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:50.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:01:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:01:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:01:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:01:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3254: done (4 left) 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3257: done (3 left) 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3258: done (2 left) 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3259: done (1 left) 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3260: done (0 left) 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3261: rmattr oid 44 current snap is 300 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3261: done (0 left) 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3262: rmattr oid 50 current snap is 300 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3262: done (0 left) 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3263: read oid 45 snap -1 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3263: expect (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3264: rmattr oid 23 current snap is 300 2024-09-21T10:01:51.604 INFO:tasks.rados.rados.0.smithi033.stdout:3264: done (1 left) 2024-09-21T10:01:51.605 INFO:tasks.rados.rados.0.smithi033.stdout:3265: snap_create 2024-09-21T10:01:51.633 INFO:tasks.rados.rados.0.smithi033.stdout:3263: expect (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:01:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:51 smithi033 ceph-mon[96568]: pgmap v649: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 72 KiB/s wr, 10 op/s 2024-09-21T10:01:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:51 smithi033 ceph-mon[96568]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T10:01:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:51 smithi033 ceph-mon[98546]: pgmap v649: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 72 KiB/s wr, 10 op/s 2024-09-21T10:01:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:51 smithi033 ceph-mon[98546]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T10:01:51.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:01:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:01:51] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-09-21T10:01:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:51 smithi105 ceph-mon[86649]: pgmap v649: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 72 KiB/s wr, 10 op/s 2024-09-21T10:01:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:51 smithi105 ceph-mon[86649]: osdmap e679: 8 total, 8 up, 8 in 2024-09-21T10:01:52.608 INFO:tasks.rados.rados.0.smithi033.stdout:3263: done (1 left) 2024-09-21T10:01:52.609 INFO:tasks.rados.rados.0.smithi033.stdout:3265: done (0 left) 2024-09-21T10:01:52.609 INFO:tasks.rados.rados.0.smithi033.stdout:3266: snap_remove snap 296 2024-09-21T10:01:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:52 smithi033 ceph-mon[96568]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T10:01:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:52 smithi033 ceph-mon[98546]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T10:01:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:52 smithi105 ceph-mon[86649]: osdmap e680: 8 total, 8 up, 8 in 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3266: done (0 left) 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3267: read oid 1 snap -1 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3267: expect (ObjNum 986 snap 287 seq_num 986) 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3268: copy_from oid 12 from oid 10 current snap is 301 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3269: read oid 45 snap -1 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3269: expect (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3270: copy_from oid 28 from oid 16 current snap is 301 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3271: read oid 2 snap 300 2024-09-21T10:01:53.551 INFO:tasks.rados.rados.0.smithi033.stdout:3271: expect deleted 2024-09-21T10:01:53.552 INFO:tasks.rados.rados.0.smithi033.stdout:3272: snap_create 2024-09-21T10:01:53.584 INFO:tasks.rados.rados.0.smithi033.stdout:3268: finishing copy_from racing read to smithi03385152-12 2024-09-21T10:01:53.586 INFO:tasks.rados.rados.0.smithi033.stdout:3268: finishing copy_from to smithi03385152-12 2024-09-21T10:01:53.587 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2263 (ObjNum 994 snap 288 seq_num 994) dirty exists 2024-09-21T10:01:53.589 INFO:tasks.rados.rados.0.smithi033.stdout:3267: expect (ObjNum 986 snap 287 seq_num 986) 2024-09-21T10:01:53.650 INFO:tasks.rados.rados.0.smithi033.stdout:3270: finishing copy_from racing read to smithi03385152-28 2024-09-21T10:01:53.651 INFO:tasks.rados.rados.0.smithi033.stdout:3270: finishing copy_from to smithi03385152-28 2024-09-21T10:01:53.651 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1643 (ObjNum 987 snap 288 seq_num 987) dirty exists 2024-09-21T10:01:53.651 INFO:tasks.rados.rados.0.smithi033.stdout:3269: expect (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:01:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:53 smithi033 ceph-mon[96568]: pgmap v652: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 148 KiB/s wr, 10 op/s 2024-09-21T10:01:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:53 smithi033 ceph-mon[96568]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T10:01:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:53 smithi033 ceph-mon[96568]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T10:01:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:53 smithi033 ceph-mon[98546]: pgmap v652: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 148 KiB/s wr, 10 op/s 2024-09-21T10:01:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:53 smithi033 ceph-mon[98546]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T10:01:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:53 smithi033 ceph-mon[98546]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T10:01:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:53 smithi105 ceph-mon[86649]: pgmap v652: 121 pgs: 121 active+clean; 143 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 148 KiB/s wr, 10 op/s 2024-09-21T10:01:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:53 smithi105 ceph-mon[86649]: osdmap e681: 8 total, 8 up, 8 in 2024-09-21T10:01:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:53 smithi105 ceph-mon[86649]: osdmap e682: 8 total, 8 up, 8 in 2024-09-21T10:01:54.557 INFO:tasks.rados.rados.0.smithi033.stdout:3267: done (5 left) 2024-09-21T10:01:54.557 INFO:tasks.rados.rados.0.smithi033.stdout:3268: done (4 left) 2024-09-21T10:01:54.557 INFO:tasks.rados.rados.0.smithi033.stdout:3269: done (3 left) 2024-09-21T10:01:54.557 INFO:tasks.rados.rados.0.smithi033.stdout:3270: done (2 left) 2024-09-21T10:01:54.557 INFO:tasks.rados.rados.0.smithi033.stdout:3271: done (1 left) 2024-09-21T10:01:54.558 INFO:tasks.rados.rados.0.smithi033.stdout:3272: done (0 left) 2024-09-21T10:01:54.558 INFO:tasks.rados.rados.0.smithi033.stdout:3273: setattr oid 18 current snap is 302 2024-09-21T10:01:54.558 INFO:tasks.rados.rados.0.smithi033.stdout:3274: copy_from oid 28 from oid 41 current snap is 302 2024-09-21T10:01:54.558 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 302 2024-09-21T10:01:54.558 INFO:tasks.rados.rados.0.smithi033.stdout:3275: seq_num 1018 ranges {1957888=253952} 2024-09-21T10:01:54.571 INFO:tasks.rados.rados.0.smithi033.stdout:3275: writing smithi03385152-3 from 1957888 to 2211840 tid 1 2024-09-21T10:01:54.571 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 302 2024-09-21T10:01:54.571 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1817 (ObjNum 952 snap 279 seq_num 952) dirty exists 2024-09-21T10:01:54.571 INFO:tasks.rados.rados.0.smithi033.stdout:3276: seq_num 1019 ranges {3293184=270336} 2024-09-21T10:01:54.590 INFO:tasks.rados.rados.0.smithi033.stdout:3276: writing smithi03385152-5 from 3293184 to 3563520 tid 1 2024-09-21T10:01:54.590 INFO:tasks.rados.rados.0.smithi033.stdout:3273: done (3 left) 2024-09-21T10:01:54.591 INFO:tasks.rados.rados.0.smithi033.stdout:3277: snap_remove snap 300 2024-09-21T10:01:55.558 INFO:tasks.rados.rados.0.smithi033.stdout:3277: done (3 left) 2024-09-21T10:01:55.558 INFO:tasks.rados.rados.0.smithi033.stdout:3278: rollback oid 49 current snap is 302 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 49 to 292 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3274: finishing copy_from racing read to smithi03385152-28 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3274: finishing copy_from to smithi03385152-28 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1826 (ObjNum 999 snap 292 seq_num 999) dirty exists 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3275: finishing write tid 1 to smithi03385152-3 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3275: finishing write tid 2 to smithi03385152-3 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3275: finishing write tid 3 to smithi03385152-3 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2377 (ObjNum 1018 snap 302 seq_num 1018) dirty exists 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3275: left oid 3 (ObjNum 1018 snap 302 seq_num 1018) 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3276: finishing write tid 1 to smithi03385152-5 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3276: finishing write tid 2 to smithi03385152-5 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3276: finishing write tid 3 to smithi03385152-5 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 1068 (ObjNum 1019 snap 302 seq_num 1019) dirty exists 2024-09-21T10:01:55.559 INFO:tasks.rados.rados.0.smithi033.stdout:3276: left oid 5 (ObjNum 1019 snap 302 seq_num 1019) 2024-09-21T10:01:55.560 INFO:tasks.rados.rados.0.smithi033.stdout:3274: done (3 left) 2024-09-21T10:01:55.560 INFO:tasks.rados.rados.0.smithi033.stdout:3275: done (2 left) 2024-09-21T10:01:55.560 INFO:tasks.rados.rados.0.smithi033.stdout:3276: done (1 left) 2024-09-21T10:01:55.560 INFO:tasks.rados.rados.0.smithi033.stdout:3279: setattr oid 16 current snap is 302 2024-09-21T10:01:55.560 INFO:tasks.rados.rados.0.smithi033.stdout:3280: copy_from oid 44 from oid 34 current snap is 302 2024-09-21T10:01:55.560 INFO:tasks.rados.rados.0.smithi033.stdout:3281: read oid 15 snap -1 2024-09-21T10:01:55.561 INFO:tasks.rados.rados.0.smithi033.stdout:3281: expect (ObjNum 1006 snap 295 seq_num 1006) 2024-09-21T10:01:55.561 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 302 2024-09-21T10:01:55.561 INFO:tasks.rados.rados.0.smithi033.stdout:3282: seq_num 1020 ranges {7118848=286720} 2024-09-21T10:01:55.602 INFO:tasks.rados.rados.0.smithi033.stdout:3282: writing smithi03385152-13 from 7118848 to 7405568 tid 1 2024-09-21T10:01:55.602 INFO:tasks.rados.rados.0.smithi033.stdout:3283: rollback oid 19 current snap is 302 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 19 to 301 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:3278: finishing rollback tid 0 to smithi03385152-49 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:3278: finishing rollback tid 1 to smithi03385152-49 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:3278: finishing rollback tid 2 to smithi03385152-49 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 2353 (ObjNum 987 snap 288 seq_num 987) dirty exists 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2385 (ObjNum 987 snap 288 seq_num 987) dirty exists 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:3280: finishing copy_from racing read to smithi03385152-44 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:3280: finishing copy_from to smithi03385152-44 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 2373 (ObjNum 1012 snap 299 seq_num 1012) dirty exists 2024-09-21T10:01:55.603 INFO:tasks.rados.rados.0.smithi033.stdout:3281: expect (ObjNum 1006 snap 295 seq_num 1006) 2024-09-21T10:01:55.897 INFO:tasks.rados.rados.0.smithi033.stdout:3283: finishing rollback tid 0 to smithi03385152-19 2024-09-21T10:01:55.897 INFO:tasks.rados.rados.0.smithi033.stdout:3283: finishing rollback tid 1 to smithi03385152-19 2024-09-21T10:01:55.897 INFO:tasks.rados.rados.0.smithi033.stdout:3283: finishing rollback tid 2 to smithi03385152-19 2024-09-21T10:01:55.897 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2269 (ObjNum 1016 snap 299 seq_num 1016) dirty exists 2024-09-21T10:01:55.897 INFO:tasks.rados.rados.0.smithi033.stdout:3282: finishing write tid 1 to smithi03385152-13 2024-09-21T10:01:55.897 INFO:tasks.rados.rados.0.smithi033.stdout:3282: finishing write tid 2 to smithi03385152-13 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3282: finishing write tid 3 to smithi03385152-13 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2388 (ObjNum 1020 snap 302 seq_num 1020) dirty exists 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3282: left oid 13 (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3278: done (5 left) 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3279: done (4 left) 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3280: done (3 left) 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3281: done (2 left) 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3282: done (1 left) 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3283: done (0 left) 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3284: read oid 6 snap -1 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3284: expect deleted 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 302 2024-09-21T10:01:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3285: seq_num 1021 ranges {6004736=303104} 2024-09-21T10:01:55.931 INFO:tasks.rados.rados.0.smithi033.stdout:3285: writing smithi03385152-45 from 6004736 to 6307840 tid 1 2024-09-21T10:01:55.931 INFO:tasks.rados.rados.0.smithi033.stdout:3286: copy_from oid 15 from oid 4 current snap is 302 2024-09-21T10:01:55.931 INFO:tasks.rados.rados.0.smithi033.stdout:3284: done (2 left) 2024-09-21T10:01:55.931 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 302 2024-09-21T10:01:55.931 INFO:tasks.rados.rados.0.smithi033.stdout:3287: seq_num 1022 ranges {2260992=319488} 2024-09-21T10:01:55.945 INFO:tasks.rados.rados.0.smithi033.stdout:3287: writing smithi03385152-10 from 2260992 to 2580480 tid 1 2024-09-21T10:01:55.945 INFO:tasks.rados.rados.0.smithi033.stdout:3288: read oid 39 snap -1 2024-09-21T10:01:55.946 INFO:tasks.rados.rados.0.smithi033.stdout:3288: expect deleted 2024-09-21T10:01:55.946 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 302 2024-09-21T10:01:55.946 INFO:tasks.rados.rados.0.smithi033.stdout:3285: finishing write tid 1 to smithi03385152-45 2024-09-21T10:01:55.946 INFO:tasks.rados.rados.0.smithi033.stdout:3285: finishing write tid 2 to smithi03385152-45 2024-09-21T10:01:55.946 INFO:tasks.rados.rados.0.smithi033.stdout:3289: seq_num 1023 ranges {9363456=335872} 2024-09-21T10:01:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:55 smithi033 ceph-mon[98546]: pgmap v655: 121 pgs: 121 active+clean; 151 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.9 MiB/s wr, 6 op/s 2024-09-21T10:01:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:55 smithi033 ceph-mon[98546]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T10:01:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:55 smithi033 ceph-mon[96568]: pgmap v655: 121 pgs: 121 active+clean; 151 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.9 MiB/s wr, 6 op/s 2024-09-21T10:01:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:55 smithi033 ceph-mon[96568]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T10:01:55.999 INFO:tasks.rados.rados.0.smithi033.stdout:3289: writing smithi03385152-31 from 9363456 to 9699328 tid 1 2024-09-21T10:01:55.999 INFO:tasks.rados.rados.0.smithi033.stdout:3290: rollback oid 20 current snap is 302 2024-09-21T10:01:55.999 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 292 2024-09-21T10:01:55.999 INFO:tasks.rados.rados.0.smithi033.stdout:3285: finishing write tid 3 to smithi03385152-45 2024-09-21T10:01:55.999 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2379 (ObjNum 1021 snap 302 seq_num 1021) dirty exists 2024-09-21T10:01:55.999 INFO:tasks.rados.rados.0.smithi033.stdout:3285: left oid 45 (ObjNum 1021 snap 302 seq_num 1021) 2024-09-21T10:01:55.999 INFO:tasks.rados.rados.0.smithi033.stdout:3286: finishing copy_from racing read to smithi03385152-15 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3286: finishing copy_from to smithi03385152-15 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3287: finishing write tid 1 to smithi03385152-10 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3287: finishing write tid 2 to smithi03385152-10 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3287: finishing write tid 3 to smithi03385152-10 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 2271 (ObjNum 1022 snap 302 seq_num 1022) dirty exists 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3287: left oid 10 (ObjNum 1022 snap 302 seq_num 1022) 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3285: done (5 left) 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3286: done (4 left) 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3287: done (3 left) 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3288: done (2 left) 2024-09-21T10:01:56.000 INFO:tasks.rados.rados.0.smithi033.stdout:3291: snap_create 2024-09-21T10:01:56.002 INFO:tasks.rados.rados.0.smithi033.stdout:3290: finishing rollback tid 0 to smithi03385152-20 2024-09-21T10:01:56.002 INFO:tasks.rados.rados.0.smithi033.stdout:3290: finishing rollback tid 1 to smithi03385152-20 2024-09-21T10:01:56.002 INFO:tasks.rados.rados.0.smithi033.stdout:3290: finishing rollback tid 2 to smithi03385152-20 2024-09-21T10:01:56.002 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1631 (ObjNum 989 snap 288 seq_num 989) dirty exists 2024-09-21T10:01:56.003 INFO:tasks.rados.rados.0.smithi033.stdout:3289: finishing write tid 1 to smithi03385152-31 2024-09-21T10:01:56.003 INFO:tasks.rados.rados.0.smithi033.stdout:3289: finishing write tid 2 to smithi03385152-31 2024-09-21T10:01:56.004 INFO:tasks.rados.rados.0.smithi033.stdout:3289: finishing write tid 3 to smithi03385152-31 2024-09-21T10:01:56.004 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2391 (ObjNum 1023 snap 302 seq_num 1023) dirty exists 2024-09-21T10:01:56.004 INFO:tasks.rados.rados.0.smithi033.stdout:3289: left oid 31 (ObjNum 1023 snap 302 seq_num 1023) 2024-09-21T10:01:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:55 smithi105 ceph-mon[86649]: pgmap v655: 121 pgs: 121 active+clean; 151 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.9 MiB/s wr, 6 op/s 2024-09-21T10:01:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:55 smithi105 ceph-mon[86649]: osdmap e683: 8 total, 8 up, 8 in 2024-09-21T10:01:56.565 INFO:tasks.rados.rados.0.smithi033.stdout:3289: done (2 left) 2024-09-21T10:01:56.565 INFO:tasks.rados.rados.0.smithi033.stdout:3290: done (1 left) 2024-09-21T10:01:56.565 INFO:tasks.rados.rados.0.smithi033.stdout:3291: done (0 left) 2024-09-21T10:01:56.565 INFO:tasks.rados.rados.0.smithi033.stdout:3292: snap_remove snap 302 2024-09-21T10:01:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:56 smithi033 ceph-mon[98546]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T10:01:56.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:56 smithi033 ceph-mon[96568]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T10:01:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:56 smithi105 ceph-mon[86649]: osdmap e684: 8 total, 8 up, 8 in 2024-09-21T10:01:57.567 INFO:tasks.rados.rados.0.smithi033.stdout:3292: done (0 left) 2024-09-21T10:01:57.567 INFO:tasks.rados.rados.0.smithi033.stdout:3293: delete oid 16 current snap is 303 2024-09-21T10:01:57.573 INFO:tasks.rados.rados.0.smithi033.stdout:3293: done (0 left) 2024-09-21T10:01:57.573 INFO:tasks.rados.rados.0.smithi033.stdout:3294: read oid 10 snap -1 2024-09-21T10:01:57.574 INFO:tasks.rados.rados.0.smithi033.stdout:3294: expect (ObjNum 1022 snap 302 seq_num 1022) 2024-09-21T10:01:57.574 INFO:tasks.rados.rados.0.smithi033.stdout:3295: rmattr oid 43 current snap is 303 2024-09-21T10:01:57.574 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 303 2024-09-21T10:01:57.574 INFO:tasks.rados.rados.0.smithi033.stdout:3296: seq_num 1025 ranges {2260992=368640} 2024-09-21T10:01:57.589 INFO:tasks.rados.rados.0.smithi033.stdout:3296: writing smithi03385152-12 from 2260992 to 2629632 tid 1 2024-09-21T10:01:57.589 INFO:tasks.rados.rados.0.smithi033.stdout:3297: rmattr oid 38 current snap is 303 2024-09-21T10:01:57.589 INFO:tasks.rados.rados.0.smithi033.stdout:3297: done (3 left) 2024-09-21T10:01:57.589 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 303 2024-09-21T10:01:57.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 1535 (ObjNum 1017 snap 299 seq_num 1017) dirty exists 2024-09-21T10:01:57.589 INFO:tasks.rados.rados.0.smithi033.stdout:3298: seq_num 1026 ranges {2113536=385024} 2024-09-21T10:01:57.602 INFO:tasks.rados.rados.0.smithi033.stdout:3298: writing smithi03385152-26 from 2113536 to 2498560 tid 1 2024-09-21T10:01:57.602 INFO:tasks.rados.rados.0.smithi033.stdout:3295: done (3 left) 2024-09-21T10:01:57.603 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 303 2024-09-21T10:01:57.603 INFO:tasks.rados.rados.0.smithi033.stdout:3299: seq_num 1027 ranges {4210688=409600} 2024-09-21T10:01:57.628 INFO:tasks.rados.rados.0.smithi033.stdout:3299: writing smithi03385152-9 from 4210688 to 4620288 tid 1 2024-09-21T10:01:57.628 INFO:tasks.rados.rados.0.smithi033.stdout:3300: delete oid 46 current snap is 303 2024-09-21T10:01:57.628 INFO:tasks.rados.rados.0.smithi033.stdout:3294: expect (ObjNum 1022 snap 302 seq_num 1022) 2024-09-21T10:01:57.734 INFO:tasks.rados.rados.0.smithi033.stdout:3296: finishing write tid 1 to smithi03385152-12 2024-09-21T10:01:57.734 INFO:tasks.rados.rados.0.smithi033.stdout:3296: finishing write tid 2 to smithi03385152-12 2024-09-21T10:01:57.734 INFO:tasks.rados.rados.0.smithi033.stdout:3296: finishing write tid 3 to smithi03385152-12 2024-09-21T10:01:57.734 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2355 (ObjNum 1025 snap 303 seq_num 1025) dirty exists 2024-09-21T10:01:57.734 INFO:tasks.rados.rados.0.smithi033.stdout:3296: left oid 12 (ObjNum 1025 snap 303 seq_num 1025) 2024-09-21T10:01:57.734 INFO:tasks.rados.rados.0.smithi033.stdout:3298: finishing write tid 1 to smithi03385152-26 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3294: done (4 left) 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3296: done (3 left) 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3300: done (2 left) 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3301: read oid 31 snap -1 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3301: expect (ObjNum 1023 snap 302 seq_num 1023) 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3302: setattr oid 46 current snap is 303 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3298: finishing write tid 2 to smithi03385152-26 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3298: finishing write tid 3 to smithi03385152-26 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 2357 (ObjNum 1026 snap 303 seq_num 1026) dirty exists 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3298: left oid 26 (ObjNum 1026 snap 303 seq_num 1026) 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3299: finishing write tid 1 to smithi03385152-9 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3299: finishing write tid 2 to smithi03385152-9 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3299: finishing write tid 3 to smithi03385152-9 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 1831 (ObjNum 1027 snap 303 seq_num 1027) dirty exists 2024-09-21T10:01:57.735 INFO:tasks.rados.rados.0.smithi033.stdout:3299: left oid 9 (ObjNum 1027 snap 303 seq_num 1027) 2024-09-21T10:01:57.736 INFO:tasks.rados.rados.0.smithi033.stdout:3298: done (3 left) 2024-09-21T10:01:57.736 INFO:tasks.rados.rados.0.smithi033.stdout:3299: done (2 left) 2024-09-21T10:01:57.736 INFO:tasks.rados.rados.0.smithi033.stdout:3303: read oid 35 snap -1 2024-09-21T10:01:57.736 INFO:tasks.rados.rados.0.smithi033.stdout:3303: expect deleted 2024-09-21T10:01:57.736 INFO:tasks.rados.rados.0.smithi033.stdout:3304: snap_remove snap 301 2024-09-21T10:01:57.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:57 smithi033 ceph-mon[98546]: pgmap v658: 121 pgs: 121 active+clean; 151 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-21T10:01:57.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:57 smithi033 ceph-mon[98546]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T10:01:57.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:57 smithi033 ceph-mon[96568]: pgmap v658: 121 pgs: 121 active+clean; 151 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-21T10:01:57.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:57 smithi033 ceph-mon[96568]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T10:01:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:57 smithi105 ceph-mon[86649]: pgmap v658: 121 pgs: 121 active+clean; 151 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 12 op/s 2024-09-21T10:01:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:57 smithi105 ceph-mon[86649]: osdmap e685: 8 total, 8 up, 8 in 2024-09-21T10:01:58.552 INFO:tasks.rados.rados.0.smithi033.stdout:3304: done (3 left) 2024-09-21T10:01:58.552 INFO:tasks.rados.rados.0.smithi033.stdout:3305: read oid 45 snap -1 2024-09-21T10:01:58.552 INFO:tasks.rados.rados.0.smithi033.stdout:3305: expect (ObjNum 1021 snap 302 seq_num 1021) 2024-09-21T10:01:58.552 INFO:tasks.rados.rados.0.smithi033.stdout:3306: rollback oid 40 current snap is 303 2024-09-21T10:01:58.552 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 40 to 292 2024-09-21T10:01:58.552 INFO:tasks.rados.rados.0.smithi033.stdout:3307: snap_remove snap 299 2024-09-21T10:01:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:58 smithi033 ceph-mon[98546]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T10:01:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:58 smithi033 ceph-mon[98546]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T10:01:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:58 smithi033 ceph-mon[96568]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T10:01:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:58 smithi033 ceph-mon[96568]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T10:01:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:58 smithi105 ceph-mon[86649]: osdmap e686: 8 total, 8 up, 8 in 2024-09-21T10:01:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:58 smithi105 ceph-mon[86649]: osdmap e687: 8 total, 8 up, 8 in 2024-09-21T10:01:59.554 INFO:tasks.rados.rados.0.smithi033.stdout:3307: done (5 left) 2024-09-21T10:01:59.554 INFO:tasks.rados.rados.0.smithi033.stdout:3308: copy_from oid 9 from oid 49 current snap is 303 2024-09-21T10:01:59.554 INFO:tasks.rados.rados.0.smithi033.stdout:3309: rollback oid 41 current snap is 303 2024-09-21T10:01:59.554 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 292 2024-09-21T10:01:59.554 INFO:tasks.rados.rados.0.smithi033.stdout:3310: read oid 2 snap -1 2024-09-21T10:01:59.554 INFO:tasks.rados.rados.0.smithi033.stdout:3310: expect deleted 2024-09-21T10:01:59.554 INFO:tasks.rados.rados.0.smithi033.stdout:3311: delete oid 10 current snap is 303 2024-09-21T10:01:59.555 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2381 (ObjNum 289 snap 0 seq_num 0) dirty exists 2024-09-21T10:01:59.555 INFO:tasks.rados.rados.0.smithi033.stdout:3301: expect (ObjNum 1023 snap 302 seq_num 1023) 2024-09-21T10:01:59.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:59 smithi105 ceph-mon[86649]: pgmap v661: 121 pgs: 121 active+clean; 158 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T10:01:59.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:01:59 smithi105 ceph-mon[86649]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T10:01:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:59 smithi033 ceph-mon[98546]: pgmap v661: 121 pgs: 121 active+clean; 158 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T10:01:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:01:59 smithi033 ceph-mon[98546]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T10:01:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:59 smithi033 ceph-mon[96568]: pgmap v661: 121 pgs: 121 active+clean; 158 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-21T10:01:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:01:59 smithi033 ceph-mon[96568]: osdmap e688: 8 total, 8 up, 8 in 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3306: finishing rollback tid 0 to smithi03385152-40 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3306: finishing rollback tid 1 to smithi03385152-40 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3306: finishing rollback tid 2 to smithi03385152-40 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1834 (ObjNum 991 snap 288 seq_num 991) dirty exists 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3301: done (8 left) 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3302: done (7 left) 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3303: done (6 left) 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3306: done (5 left) 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3311: done (4 left) 2024-09-21T10:02:00.511 INFO:tasks.rados.rados.0.smithi033.stdout:3312: rmattr oid 38 current snap is 303 2024-09-21T10:02:00.512 INFO:tasks.rados.rados.0.smithi033.stdout:3312: done (4 left) 2024-09-21T10:02:00.512 INFO:tasks.rados.rados.0.smithi033.stdout:3313: rmattr oid 18 current snap is 303 2024-09-21T10:02:00.512 INFO:tasks.rados.rados.0.smithi033.stdout:3305: expect (ObjNum 1021 snap 302 seq_num 1021) 2024-09-21T10:02:00.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:02:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:02:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:00.865 INFO:tasks.rados.rados.0.smithi033.stdout:3309: finishing rollback tid 0 to smithi03385152-41 2024-09-21T10:02:00.865 INFO:tasks.rados.rados.0.smithi033.stdout:3309: finishing rollback tid 1 to smithi03385152-41 2024-09-21T10:02:00.865 INFO:tasks.rados.rados.0.smithi033.stdout:3309: finishing rollback tid 2 to smithi03385152-41 2024-09-21T10:02:00.865 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 1837 (ObjNum 999 snap 292 seq_num 999) dirty exists 2024-09-21T10:02:00.865 INFO:tasks.rados.rados.0.smithi033.stdout:3308: finishing copy_from racing read to smithi03385152-9 2024-09-21T10:02:00.865 INFO:tasks.rados.rados.0.smithi033.stdout:3308: finishing copy_from to smithi03385152-9 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 2353 (ObjNum 987 snap 288 seq_num 987) dirty exists 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1818 (ObjNum 952 snap 279 seq_num 952) dirty exists 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:3305: done (4 left) 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:3308: done (3 left) 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:3309: done (2 left) 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:3310: done (1 left) 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:3313: done (0 left) 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:3314: copy_from oid 41 from oid 15 current snap is 303 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 303 2024-09-21T10:02:00.866 INFO:tasks.rados.rados.0.smithi033.stdout:3315: seq_num 1030 ranges {3932160=458752} 2024-09-21T10:02:00.889 INFO:tasks.rados.rados.0.smithi033.stdout:3315: writing smithi03385152-20 from 3932160 to 4390912 tid 1 2024-09-21T10:02:00.890 INFO:tasks.rados.rados.0.smithi033.stdout:3316: read oid 8 snap -1 2024-09-21T10:02:00.890 INFO:tasks.rados.rados.0.smithi033.stdout:3316: expect (ObjNum 1001 snap 292 seq_num 1001) 2024-09-21T10:02:00.890 INFO:tasks.rados.rados.0.smithi033.stdout:3317: read oid 24 snap -1 2024-09-21T10:02:00.890 INFO:tasks.rados.rados.0.smithi033.stdout:3317: expect (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:02:00.890 INFO:tasks.rados.rados.0.smithi033.stdout:3318: snap_remove snap 292 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3318: done (4 left) 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3319: copy_from oid 46 from oid 26 current snap is 303 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3320: rmattr oid 34 current snap is 303 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3320: done (5 left) 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3321: rollback oid 21 current snap is 303 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3321: done (5 left) 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3322: snap_create 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3314: finishing copy_from racing read to smithi03385152-41 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:3314: finishing copy_from to smithi03385152-41 2024-09-21T10:02:01.595 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:01.596 INFO:tasks.rados.rados.0.smithi033.stdout:3315: finishing write tid 1 to smithi03385152-20 2024-09-21T10:02:01.596 INFO:tasks.rados.rados.0.smithi033.stdout:3315: finishing write tid 2 to smithi03385152-20 2024-09-21T10:02:01.596 INFO:tasks.rados.rados.0.smithi033.stdout:3317: expect (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:02:01.659 INFO:tasks.rados.rados.0.smithi033.stdout:3315: finishing write tid 3 to smithi03385152-20 2024-09-21T10:02:01.659 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 1633 (ObjNum 1030 snap 303 seq_num 1030) dirty exists 2024-09-21T10:02:01.659 INFO:tasks.rados.rados.0.smithi033.stdout:3315: left oid 20 (ObjNum 1030 snap 303 seq_num 1030) 2024-09-21T10:02:01.659 INFO:tasks.rados.rados.0.smithi033.stdout:3316: expect (ObjNum 1001 snap 292 seq_num 1001) 2024-09-21T10:02:01.933 INFO:tasks.rados.rados.0.smithi033.stdout:3319: finishing copy_from racing read to smithi03385152-46 2024-09-21T10:02:01.933 INFO:tasks.rados.rados.0.smithi033.stdout:3319: finishing copy_from to smithi03385152-46 2024-09-21T10:02:01.933 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2357 (ObjNum 1026 snap 303 seq_num 1026) dirty exists 2024-09-21T10:02:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:01 smithi033 ceph-mon[98546]: pgmap v664: 121 pgs: 121 active+clean; 159 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T10:02:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:02:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:02:01] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-09-21T10:02:01.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:01 smithi033 ceph-mon[96568]: pgmap v664: 121 pgs: 121 active+clean; 159 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T10:02:01.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:01 smithi105 ceph-mon[86649]: pgmap v664: 121 pgs: 121 active+clean; 159 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-21T10:02:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:02.600 INFO:tasks.rados.rados.0.smithi033.stdout:3314: done (5 left) 2024-09-21T10:02:02.600 INFO:tasks.rados.rados.0.smithi033.stdout:3315: done (4 left) 2024-09-21T10:02:02.600 INFO:tasks.rados.rados.0.smithi033.stdout:3316: done (3 left) 2024-09-21T10:02:02.600 INFO:tasks.rados.rados.0.smithi033.stdout:3317: done (2 left) 2024-09-21T10:02:02.601 INFO:tasks.rados.rados.0.smithi033.stdout:3319: done (1 left) 2024-09-21T10:02:02.601 INFO:tasks.rados.rados.0.smithi033.stdout:3322: done (0 left) 2024-09-21T10:02:02.601 INFO:tasks.rados.rados.0.smithi033.stdout:3323: copy_from oid 46 from oid 15 current snap is 304 2024-09-21T10:02:02.601 INFO:tasks.rados.rados.0.smithi033.stdout:3324: read oid 25 snap -1 2024-09-21T10:02:02.601 INFO:tasks.rados.rados.0.smithi033.stdout:3324: expect (ObjNum 848 snap 249 seq_num 848) 2024-09-21T10:02:02.601 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 304 2024-09-21T10:02:02.601 INFO:tasks.rados.rados.0.smithi033.stdout:3325: seq_num 1031 ranges {0=475136} 2024-09-21T10:02:02.603 INFO:tasks.rados.rados.0.smithi033.stdout:3325: writing smithi03385152-16 from 0 to 475136 tid 1 2024-09-21T10:02:02.603 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 304 2024-09-21T10:02:02.603 INFO:tasks.rados.rados.0.smithi033.stdout:3326: seq_num 1032 ranges {2400256=491520} 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3326: writing smithi03385152-28 from 2400256 to 2891776 tid 1 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3327: read oid 45 snap -1 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3327: expect (ObjNum 1021 snap 302 seq_num 1021) 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3328: copy_from oid 40 from oid 50 current snap is 304 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3325: finishing write tid 1 to smithi03385152-16 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3325: finishing write tid 2 to smithi03385152-16 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3325: finishing write tid 3 to smithi03385152-16 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2414 (ObjNum 1031 snap 304 seq_num 1031) dirty exists 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3325: left oid 16 (ObjNum 1031 snap 304 seq_num 1031) 2024-09-21T10:02:02.620 INFO:tasks.rados.rados.0.smithi033.stdout:3323: finishing copy_from racing read to smithi03385152-46 2024-09-21T10:02:02.621 INFO:tasks.rados.rados.0.smithi033.stdout:3323: finishing copy_from to smithi03385152-46 2024-09-21T10:02:02.621 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:02.621 INFO:tasks.rados.rados.0.smithi033.stdout:3323: done (5 left) 2024-09-21T10:02:02.621 INFO:tasks.rados.rados.0.smithi033.stdout:3325: done (4 left) 2024-09-21T10:02:02.621 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 304 2024-09-21T10:02:02.621 INFO:tasks.rados.rados.0.smithi033.stdout:3329: seq_num 1033 ranges {106496=507904} 2024-09-21T10:02:02.624 INFO:tasks.rados.rados.0.smithi033.stdout:3329: writing smithi03385152-27 from 106496 to 614400 tid 1 2024-09-21T10:02:02.624 INFO:tasks.rados.rados.0.smithi033.stdout:3330: copy_from oid 44 from oid 34 current snap is 304 2024-09-21T10:02:02.624 INFO:tasks.rados.rados.0.smithi033.stdout:3331: snap_remove snap 303 2024-09-21T10:02:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:02 smithi033 ceph-mon[98546]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T10:02:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:02 smithi033 ceph-mon[96568]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T10:02:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:02 smithi105 ceph-mon[86649]: osdmap e689: 8 total, 8 up, 8 in 2024-09-21T10:02:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:3331: done (6 left) 2024-09-21T10:02:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:3332: rollback oid 35 current snap is 304 2024-09-21T10:02:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:3332: done (6 left) 2024-09-21T10:02:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:3333: snap_create 2024-09-21T10:02:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:3326: finishing write tid 1 to smithi03385152-28 2024-09-21T10:02:03.554 INFO:tasks.rados.rados.0.smithi033.stdout:3324: expect (ObjNum 848 snap 249 seq_num 848) 2024-09-21T10:02:03.909 INFO:tasks.rados.rados.0.smithi033.stdout:3326: finishing write tid 2 to smithi03385152-28 2024-09-21T10:02:03.909 INFO:tasks.rados.rados.0.smithi033.stdout:3326: finishing write tid 3 to smithi03385152-28 2024-09-21T10:02:03.909 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1828 (ObjNum 1032 snap 304 seq_num 1032) dirty exists 2024-09-21T10:02:03.909 INFO:tasks.rados.rados.0.smithi033.stdout:3326: left oid 28 (ObjNum 1032 snap 304 seq_num 1032) 2024-09-21T10:02:03.909 INFO:tasks.rados.rados.0.smithi033.stdout:3329: finishing write tid 1 to smithi03385152-27 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:3329: finishing write tid 2 to smithi03385152-27 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:3329: finishing write tid 3 to smithi03385152-27 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 1632 (ObjNum 1033 snap 304 seq_num 1033) dirty exists 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:3329: left oid 27 (ObjNum 1033 snap 304 seq_num 1033) 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:3330: finishing copy_from racing read to smithi03385152-44 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:3330: finishing copy_from to smithi03385152-44 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 2373 (ObjNum 1012 snap 299 seq_num 1012) dirty exists 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:3328: finishing copy_from racing read to smithi03385152-40 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:3328: finishing copy_from to smithi03385152-40 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1800 (ObjNum 965 snap 283 seq_num 965) dirty exists 2024-09-21T10:02:03.910 INFO:tasks.rados.rados.0.smithi033.stdout:3327: expect (ObjNum 1021 snap 302 seq_num 1021) 2024-09-21T10:02:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:03 smithi033 ceph-mon[98546]: pgmap v666: 121 pgs: 121 active+clean; 158 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 512 KiB/s wr, 18 op/s 2024-09-21T10:02:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:03 smithi033 ceph-mon[98546]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T10:02:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:03 smithi033 ceph-mon[98546]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T10:02:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:03 smithi033 ceph-mon[96568]: pgmap v666: 121 pgs: 121 active+clean; 158 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 512 KiB/s wr, 18 op/s 2024-09-21T10:02:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:03 smithi033 ceph-mon[96568]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T10:02:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:03 smithi033 ceph-mon[96568]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T10:02:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:03 smithi105 ceph-mon[86649]: pgmap v666: 121 pgs: 121 active+clean; 158 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 512 KiB/s wr, 18 op/s 2024-09-21T10:02:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:03 smithi105 ceph-mon[86649]: osdmap e690: 8 total, 8 up, 8 in 2024-09-21T10:02:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:03 smithi105 ceph-mon[86649]: osdmap e691: 8 total, 8 up, 8 in 2024-09-21T10:02:04.554 INFO:tasks.rados.rados.0.smithi033.stdout:3324: done (6 left) 2024-09-21T10:02:04.554 INFO:tasks.rados.rados.0.smithi033.stdout:3326: done (5 left) 2024-09-21T10:02:04.554 INFO:tasks.rados.rados.0.smithi033.stdout:3327: done (4 left) 2024-09-21T10:02:04.554 INFO:tasks.rados.rados.0.smithi033.stdout:3328: done (3 left) 2024-09-21T10:02:04.554 INFO:tasks.rados.rados.0.smithi033.stdout:3329: done (2 left) 2024-09-21T10:02:04.554 INFO:tasks.rados.rados.0.smithi033.stdout:3330: done (1 left) 2024-09-21T10:02:04.554 INFO:tasks.rados.rados.0.smithi033.stdout:3333: done (0 left) 2024-09-21T10:02:04.554 INFO:tasks.rados.rados.0.smithi033.stdout:3334: delete oid 6 current snap is 305 2024-09-21T10:02:04.558 INFO:tasks.rados.rados.0.smithi033.stdout:3334: done (0 left) 2024-09-21T10:02:04.558 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 305 2024-09-21T10:02:04.558 INFO:tasks.rados.rados.0.smithi033.stdout:3335: seq_num 1035 ranges {155648=540672} 2024-09-21T10:02:04.562 INFO:tasks.rados.rados.0.smithi033.stdout:3335: writing smithi03385152-34 from 155648 to 696320 tid 1 2024-09-21T10:02:04.562 INFO:tasks.rados.rados.0.smithi033.stdout:3336: read oid 41 snap 304 2024-09-21T10:02:04.562 INFO:tasks.rados.rados.0.smithi033.stdout:3336: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:04.562 INFO:tasks.rados.rados.0.smithi033.stdout:3337: copy_from oid 38 from oid 44 current snap is 305 2024-09-21T10:02:04.562 INFO:tasks.rados.rados.0.smithi033.stdout:3338: delete oid 10 current snap is 305 2024-09-21T10:02:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:3338: done (3 left) 2024-09-21T10:02:04.564 INFO:tasks.rados.rados.0.smithi033.stdout:3339: rollback oid 1 current snap is 305 2024-09-21T10:02:04.565 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 304 2024-09-21T10:02:04.565 INFO:tasks.rados.rados.0.smithi033.stdout:3340: read oid 25 snap -1 2024-09-21T10:02:04.565 INFO:tasks.rados.rados.0.smithi033.stdout:3340: expect (ObjNum 848 snap 249 seq_num 848) 2024-09-21T10:02:04.565 INFO:tasks.rados.rados.0.smithi033.stdout:3341: copy_from oid 43 from oid 13 current snap is 305 2024-09-21T10:02:04.565 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 305 2024-09-21T10:02:04.565 INFO:tasks.rados.rados.0.smithi033.stdout:3342: seq_num 1037 ranges {2113536=573440} 2024-09-21T10:02:04.580 INFO:tasks.rados.rados.0.smithi033.stdout:3342: writing smithi03385152-30 from 2113536 to 2686976 tid 1 2024-09-21T10:02:04.580 INFO:tasks.rados.rados.0.smithi033.stdout:3343: rollback oid 16 current snap is 305 2024-09-21T10:02:04.580 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 16 to 304 2024-09-21T10:02:04.580 INFO:tasks.rados.rados.0.smithi033.stdout:3335: finishing write tid 1 to smithi03385152-34 2024-09-21T10:02:04.580 INFO:tasks.rados.rados.0.smithi033.stdout:3339: finishing rollback tid 0 to smithi03385152-1 2024-09-21T10:02:04.580 INFO:tasks.rados.rados.0.smithi033.stdout:3339: finishing rollback tid 1 to smithi03385152-1 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:3344: rollback oid 2 current snap is 305 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 2 to 304 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:3335: finishing write tid 2 to smithi03385152-34 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:3339: finishing rollback tid 2 to smithi03385152-1 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1627 (ObjNum 986 snap 287 seq_num 986) dirty exists 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:3335: finishing write tid 3 to smithi03385152-34 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 2383 (ObjNum 1035 snap 305 seq_num 1035) dirty exists 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:3335: left oid 34 (ObjNum 1035 snap 305 seq_num 1035) 2024-09-21T10:02:04.581 INFO:tasks.rados.rados.0.smithi033.stdout:3336: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:04.639 INFO:tasks.rados.rados.0.smithi033.stdout:3337: finishing copy_from racing read to smithi03385152-38 2024-09-21T10:02:04.639 INFO:tasks.rados.rados.0.smithi033.stdout:3337: finishing copy_from to smithi03385152-38 2024-09-21T10:02:04.639 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2373 (ObjNum 1012 snap 299 seq_num 1012) dirty exists 2024-09-21T10:02:04.639 INFO:tasks.rados.rados.0.smithi033.stdout:3340: expect (ObjNum 848 snap 249 seq_num 848) 2024-09-21T10:02:04.991 INFO:tasks.rados.rados.0.smithi033.stdout:3344: finishing rollback tid 1 to smithi03385152-2 2024-09-21T10:02:04.991 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 0 (ObjNum 65 snap 0 seq_num 114196896) dirty dne 2024-09-21T10:02:04.991 INFO:tasks.rados.rados.0.smithi033.stdout:3342: finishing write tid 1 to smithi03385152-30 2024-09-21T10:02:04.991 INFO:tasks.rados.rados.0.smithi033.stdout:3342: finishing write tid 2 to smithi03385152-30 2024-09-21T10:02:04.991 INFO:tasks.rados.rados.0.smithi033.stdout:3343: finishing rollback tid 0 to smithi03385152-16 2024-09-21T10:02:04.991 INFO:tasks.rados.rados.0.smithi033.stdout:3342: finishing write tid 3 to smithi03385152-30 2024-09-21T10:02:04.991 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2417 (ObjNum 1037 snap 305 seq_num 1037) dirty exists 2024-09-21T10:02:04.991 INFO:tasks.rados.rados.0.smithi033.stdout:3342: left oid 30 (ObjNum 1037 snap 305 seq_num 1037) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3343: finishing rollback tid 1 to smithi03385152-16 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3343: finishing rollback tid 2 to smithi03385152-16 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2421 (ObjNum 1031 snap 304 seq_num 1031) dirty exists 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3341: finishing copy_from racing read to smithi03385152-43 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3335: done (8 left) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3336: done (7 left) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3337: done (6 left) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3339: done (5 left) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3340: done (4 left) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3342: done (3 left) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3343: done (2 left) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3344: done (1 left) 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3345: snap_create 2024-09-21T10:02:04.992 INFO:tasks.rados.rados.0.smithi033.stdout:3341: finishing copy_from to smithi03385152-43 2024-09-21T10:02:04.993 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 2388 (ObjNum 1020 snap 302 seq_num 1020) dirty exists 2024-09-21T10:02:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3341: done (1 left) 2024-09-21T10:02:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3345: done (0 left) 2024-09-21T10:02:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 306 2024-09-21T10:02:05.604 INFO:tasks.rados.rados.0.smithi033.stdout:3346: seq_num 1038 ranges {2686976=589824} 2024-09-21T10:02:05.622 INFO:tasks.rados.rados.0.smithi033.stdout:3346: writing smithi03385152-30 from 2686976 to 3276800 tid 1 2024-09-21T10:02:05.622 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 306 2024-09-21T10:02:05.622 INFO:tasks.rados.rados.0.smithi033.stdout:3347: seq_num 1039 ranges {155648=606208} 2024-09-21T10:02:05.626 INFO:tasks.rados.rados.0.smithi033.stdout:3347: writing smithi03385152-38 from 155648 to 761856 tid 1 2024-09-21T10:02:05.626 INFO:tasks.rados.rados.0.smithi033.stdout:3348: snap_create 2024-09-21T10:02:05.627 INFO:tasks.rados.rados.0.smithi033.stdout:3346: finishing write tid 1 to smithi03385152-30 2024-09-21T10:02:05.628 INFO:tasks.rados.rados.0.smithi033.stdout:3346: finishing write tid 2 to smithi03385152-30 2024-09-21T10:02:05.629 INFO:tasks.rados.rados.0.smithi033.stdout:3346: finishing write tid 3 to smithi03385152-30 2024-09-21T10:02:05.629 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2424 (ObjNum 1038 snap 306 seq_num 1038) dirty exists 2024-09-21T10:02:05.629 INFO:tasks.rados.rados.0.smithi033.stdout:3346: left oid 30 (ObjNum 1038 snap 306 seq_num 1038) 2024-09-21T10:02:05.630 INFO:tasks.rados.rados.0.smithi033.stdout:3347: finishing write tid 1 to smithi03385152-38 2024-09-21T10:02:05.631 INFO:tasks.rados.rados.0.smithi033.stdout:3347: finishing write tid 2 to smithi03385152-38 2024-09-21T10:02:05.632 INFO:tasks.rados.rados.0.smithi033.stdout:3347: finishing write tid 3 to smithi03385152-38 2024-09-21T10:02:05.632 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2375 (ObjNum 1039 snap 306 seq_num 1039) dirty exists 2024-09-21T10:02:05.632 INFO:tasks.rados.rados.0.smithi033.stdout:3347: left oid 38 (ObjNum 1039 snap 306 seq_num 1039) 2024-09-21T10:02:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:05 smithi033 ceph-mon[98546]: pgmap v669: 121 pgs: 121 active+clean; 154 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-09-21T10:02:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:05 smithi033 ceph-mon[98546]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T10:02:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:05 smithi033 ceph-mon[96568]: pgmap v669: 121 pgs: 121 active+clean; 154 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-09-21T10:02:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:05 smithi033 ceph-mon[96568]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T10:02:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:05 smithi105 ceph-mon[86649]: pgmap v669: 121 pgs: 121 active+clean; 154 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-09-21T10:02:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:05 smithi105 ceph-mon[86649]: osdmap e692: 8 total, 8 up, 8 in 2024-09-21T10:02:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:3346: done (2 left) 2024-09-21T10:02:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:3347: done (1 left) 2024-09-21T10:02:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:3348: done (0 left) 2024-09-21T10:02:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 307 2024-09-21T10:02:06.609 INFO:tasks.rados.rados.0.smithi033.stdout:3349: seq_num 1040 ranges {9699328=622592} 2024-09-21T10:02:06.665 INFO:tasks.rados.rados.0.smithi033.stdout:3349: writing smithi03385152-31 from 9699328 to 10321920 tid 1 2024-09-21T10:02:06.665 INFO:tasks.rados.rados.0.smithi033.stdout:3350: rmattr oid 35 current snap is 307 2024-09-21T10:02:06.665 INFO:tasks.rados.rados.0.smithi033.stdout:3350: done (1 left) 2024-09-21T10:02:06.665 INFO:tasks.rados.rados.0.smithi033.stdout:3351: read oid 29 snap 306 2024-09-21T10:02:06.666 INFO:tasks.rados.rados.0.smithi033.stdout:3351: expect (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:02:06.666 INFO:tasks.rados.rados.0.smithi033.stdout:3352: copy_from oid 11 from oid 13 current snap is 307 2024-09-21T10:02:06.666 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 307 2024-09-21T10:02:06.666 INFO:tasks.rados.rados.0.smithi033.stdout:3353: seq_num 1041 ranges {2023424=638976} 2024-09-21T10:02:06.681 INFO:tasks.rados.rados.0.smithi033.stdout:3353: writing smithi03385152-46 from 2023424 to 2662400 tid 1 2024-09-21T10:02:06.681 INFO:tasks.rados.rados.0.smithi033.stdout:3354: delete oid 18 current snap is 307 2024-09-21T10:02:06.681 INFO:tasks.rados.rados.0.smithi033.stdout:3349: finishing write tid 1 to smithi03385152-31 2024-09-21T10:02:06.681 INFO:tasks.rados.rados.0.smithi033.stdout:3349: finishing write tid 2 to smithi03385152-31 2024-09-21T10:02:06.685 INFO:tasks.rados.rados.0.smithi033.stdout:3354: done (4 left) 2024-09-21T10:02:06.685 INFO:tasks.rados.rados.0.smithi033.stdout:3355: read oid 23 snap -1 2024-09-21T10:02:06.685 INFO:tasks.rados.rados.0.smithi033.stdout:3355: expect deleted 2024-09-21T10:02:06.686 INFO:tasks.rados.rados.0.smithi033.stdout:3356: copy_from oid 39 from oid 43 current snap is 307 2024-09-21T10:02:06.686 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 307 2024-09-21T10:02:06.686 INFO:tasks.rados.rados.0.smithi033.stdout:3357: seq_num 1043 ranges {1466368=671744} 2024-09-21T10:02:06.697 INFO:tasks.rados.rados.0.smithi033.stdout:3357: writing smithi03385152-17 from 1466368 to 2138112 tid 1 2024-09-21T10:02:06.698 INFO:tasks.rados.rados.0.smithi033.stdout:3358: read oid 1 snap -1 2024-09-21T10:02:06.698 INFO:tasks.rados.rados.0.smithi033.stdout:3358: expect (ObjNum 986 snap 287 seq_num 986) 2024-09-21T10:02:06.698 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 307 2024-09-21T10:02:06.698 INFO:tasks.rados.rados.0.smithi033.stdout:3349: finishing write tid 3 to smithi03385152-31 2024-09-21T10:02:06.698 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2427 (ObjNum 1040 snap 307 seq_num 1040) dirty exists 2024-09-21T10:02:06.698 INFO:tasks.rados.rados.0.smithi033.stdout:3349: left oid 31 (ObjNum 1040 snap 307 seq_num 1040) 2024-09-21T10:02:06.698 INFO:tasks.rados.rados.0.smithi033.stdout:3359: seq_num 1044 ranges {0=688128} 2024-09-21T10:02:06.702 INFO:tasks.rados.rados.0.smithi033.stdout:3359: writing smithi03385152-18 from 0 to 688128 tid 1 2024-09-21T10:02:06.702 INFO:tasks.rados.rados.0.smithi033.stdout:3349: done (8 left) 2024-09-21T10:02:06.702 INFO:tasks.rados.rados.0.smithi033.stdout:3355: done (7 left) 2024-09-21T10:02:06.702 INFO:tasks.rados.rados.0.smithi033.stdout:3360: read oid 31 snap -1 2024-09-21T10:02:06.702 INFO:tasks.rados.rados.0.smithi033.stdout:3360: expect (ObjNum 1040 snap 307 seq_num 1040) 2024-09-21T10:02:06.703 INFO:tasks.rados.rados.0.smithi033.stdout:3361: copy_from oid 33 from oid 19 current snap is 307 2024-09-21T10:02:06.703 INFO:tasks.rados.rados.0.smithi033.stdout:3362: copy_from oid 42 from oid 25 current snap is 307 2024-09-21T10:02:06.703 INFO:tasks.rados.rados.0.smithi033.stdout:3363: read oid 5 snap -1 2024-09-21T10:02:06.703 INFO:tasks.rados.rados.0.smithi033.stdout:3363: expect (ObjNum 1019 snap 302 seq_num 1019) 2024-09-21T10:02:06.703 INFO:tasks.rados.rados.0.smithi033.stdout:3364: rollback oid 38 current snap is 307 2024-09-21T10:02:06.703 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 305 2024-09-21T10:02:06.703 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 307 2024-09-21T10:02:06.703 INFO:tasks.rados.rados.0.smithi033.stdout:3365: seq_num 1045 ranges {6307840=704512} 2024-09-21T10:02:06.745 INFO:tasks.rados.rados.0.smithi033.stdout:3365: writing smithi03385152-45 from 6307840 to 7012352 tid 1 2024-09-21T10:02:06.745 INFO:tasks.rados.rados.0.smithi033.stdout:3366: read oid 23 snap -1 2024-09-21T10:02:06.745 INFO:tasks.rados.rados.0.smithi033.stdout:3366: expect deleted 2024-09-21T10:02:06.745 INFO:tasks.rados.rados.0.smithi033.stdout:3367: delete oid 27 current snap is 307 2024-09-21T10:02:06.745 INFO:tasks.rados.rados.0.smithi033.stdout:3359: finishing write tid 1 to smithi03385152-18 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3359: finishing write tid 2 to smithi03385152-18 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3357: finishing write tid 1 to smithi03385152-17 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3357: finishing write tid 2 to smithi03385152-17 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3357: finishing write tid 3 to smithi03385152-17 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2359 (ObjNum 1043 snap 307 seq_num 1043) dirty exists 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3357: left oid 17 (ObjNum 1043 snap 307 seq_num 1043) 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3353: finishing write tid 1 to smithi03385152-46 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3359: finishing write tid 3 to smithi03385152-18 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1821 (ObjNum 1044 snap 307 seq_num 1044) dirty exists 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3359: left oid 18 (ObjNum 1044 snap 307 seq_num 1044) 2024-09-21T10:02:06.746 INFO:tasks.rados.rados.0.smithi033.stdout:3358: expect (ObjNum 986 snap 287 seq_num 986) 2024-09-21T10:02:06.809 INFO:tasks.rados.rados.0.smithi033.stdout:3351: expect (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:02:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:06 smithi033 ceph-mon[96568]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T10:02:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:06 smithi033 ceph-mon[98546]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T10:02:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:06 smithi105 ceph-mon[86649]: osdmap e693: 8 total, 8 up, 8 in 2024-09-21T10:02:07.058 INFO:tasks.rados.rados.0.smithi033.stdout:3353: finishing write tid 2 to smithi03385152-46 2024-09-21T10:02:07.058 INFO:tasks.rados.rados.0.smithi033.stdout:3353: finishing write tid 3 to smithi03385152-46 2024-09-21T10:02:07.058 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2385 (ObjNum 1041 snap 307 seq_num 1041) dirty exists 2024-09-21T10:02:07.058 INFO:tasks.rados.rados.0.smithi033.stdout:3353: left oid 46 (ObjNum 1041 snap 307 seq_num 1041) 2024-09-21T10:02:07.058 INFO:tasks.rados.rados.0.smithi033.stdout:3352: finishing copy_from racing read to smithi03385152-11 2024-09-21T10:02:07.058 INFO:tasks.rados.rados.0.smithi033.stdout:3364: finishing rollback tid 0 to smithi03385152-38 2024-09-21T10:02:07.058 INFO:tasks.rados.rados.0.smithi033.stdout:3356: finishing copy_from racing read to smithi03385152-39 2024-09-21T10:02:07.058 INFO:tasks.rados.rados.0.smithi033.stdout:3352: finishing copy_from to smithi03385152-11 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 2388 (ObjNum 1020 snap 302 seq_num 1020) dirty exists 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3361: finishing copy_from racing read to smithi03385152-33 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3351: done (14 left) 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3352: done (13 left) 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3353: done (12 left) 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3357: done (11 left) 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3358: done (10 left) 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3359: done (9 left) 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3367: done (8 left) 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3368: rollback oid 22 current snap is 307 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 22 to 306 2024-09-21T10:02:07.059 INFO:tasks.rados.rados.0.smithi033.stdout:3362: finishing copy_from racing read to smithi03385152-42 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:3364: finishing rollback tid 1 to smithi03385152-38 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:3364: finishing rollback tid 2 to smithi03385152-38 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2378 (ObjNum 1012 snap 299 seq_num 1012) dirty exists 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:3356: finishing copy_from to smithi03385152-39 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 2388 (ObjNum 1020 snap 302 seq_num 1020) dirty exists 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:3362: finishing copy_from to smithi03385152-42 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 960 (ObjNum 848 snap 249 seq_num 848) dirty exists 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:3361: finishing copy_from to smithi03385152-33 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 2269 (ObjNum 1016 snap 299 seq_num 1016) dirty exists 2024-09-21T10:02:07.060 INFO:tasks.rados.rados.0.smithi033.stdout:3363: expect (ObjNum 1019 snap 302 seq_num 1019) 2024-09-21T10:02:07.223 INFO:tasks.rados.rados.0.smithi033.stdout:3360: expect (ObjNum 1040 snap 307 seq_num 1040) 2024-09-21T10:02:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:07 smithi033 ceph-mon[96568]: pgmap v672: 121 pgs: 121 active+clean; 156 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.9 MiB/s wr, 14 op/s 2024-09-21T10:02:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:07 smithi033 ceph-mon[96568]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T10:02:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:07 smithi033 ceph-mon[98546]: pgmap v672: 121 pgs: 121 active+clean; 156 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.9 MiB/s wr, 14 op/s 2024-09-21T10:02:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:07 smithi033 ceph-mon[98546]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T10:02:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:07 smithi105 ceph-mon[86649]: pgmap v672: 121 pgs: 121 active+clean; 156 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 3.9 MiB/s wr, 14 op/s 2024-09-21T10:02:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:07 smithi105 ceph-mon[86649]: osdmap e694: 8 total, 8 up, 8 in 2024-09-21T10:02:08.345 INFO:tasks.rados.rados.0.smithi033.stdout:3365: finishing write tid 1 to smithi03385152-45 2024-09-21T10:02:08.345 INFO:tasks.rados.rados.0.smithi033.stdout:3365: finishing write tid 2 to smithi03385152-45 2024-09-21T10:02:08.345 INFO:tasks.rados.rados.0.smithi033.stdout:3365: finishing write tid 3 to smithi03385152-45 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2387 (ObjNum 1045 snap 307 seq_num 1045) dirty exists 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3365: left oid 45 (ObjNum 1045 snap 307 seq_num 1045) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3356: done (8 left) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3360: done (7 left) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3361: done (6 left) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3362: done (5 left) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3363: done (4 left) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3364: done (3 left) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3365: done (2 left) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3366: done (1 left) 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 307 2024-09-21T10:02:08.346 INFO:tasks.rados.rados.0.smithi033.stdout:3369: seq_num 1047 ranges {3276800=745472} 2024-09-21T10:02:08.367 INFO:tasks.rados.rados.0.smithi033.stdout:3369: writing smithi03385152-30 from 3276800 to 4022272 tid 1 2024-09-21T10:02:08.367 INFO:tasks.rados.rados.0.smithi033.stdout:3370: snap_create 2024-09-21T10:02:08.367 INFO:tasks.rados.rados.0.smithi033.stdout:3368: finishing rollback tid 0 to smithi03385152-22 2024-09-21T10:02:08.367 INFO:tasks.rados.rados.0.smithi033.stdout:3368: finishing rollback tid 1 to smithi03385152-22 2024-09-21T10:02:08.367 INFO:tasks.rados.rados.0.smithi033.stdout:3368: finishing rollback tid 2 to smithi03385152-22 2024-09-21T10:02:08.367 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2390 (ObjNum 983 snap 287 seq_num 983) dirty exists 2024-09-21T10:02:08.372 INFO:tasks.rados.rados.0.smithi033.stdout:3369: finishing write tid 1 to smithi03385152-30 2024-09-21T10:02:08.372 INFO:tasks.rados.rados.0.smithi033.stdout:3369: finishing write tid 2 to smithi03385152-30 2024-09-21T10:02:08.374 INFO:tasks.rados.rados.0.smithi033.stdout:3369: finishing write tid 3 to smithi03385152-30 2024-09-21T10:02:08.374 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2430 (ObjNum 1047 snap 307 seq_num 1047) dirty exists 2024-09-21T10:02:08.374 INFO:tasks.rados.rados.0.smithi033.stdout:3369: left oid 30 (ObjNum 1047 snap 307 seq_num 1047) 2024-09-21T10:02:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:3368: done (2 left) 2024-09-21T10:02:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:3369: done (1 left) 2024-09-21T10:02:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:3370: done (0 left) 2024-09-21T10:02:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:3371: rmattr oid 32 current snap is 308 2024-09-21T10:02:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:3372: rollback oid 17 current snap is 308 2024-09-21T10:02:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 304 2024-09-21T10:02:08.554 INFO:tasks.rados.rados.0.smithi033.stdout:3373: copy_from oid 9 from oid 8 current snap is 308 2024-09-21T10:02:08.555 INFO:tasks.rados.rados.0.smithi033.stdout:3374: read oid 30 snap 304 2024-09-21T10:02:08.555 INFO:tasks.rados.rados.0.smithi033.stdout:3374: expect (ObjNum 982 snap 286 seq_num 982) 2024-09-21T10:02:08.555 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 308 2024-09-21T10:02:08.555 INFO:tasks.rados.rados.0.smithi033.stdout:3375: seq_num 1048 ranges {2891776=761856} 2024-09-21T10:02:08.575 INFO:tasks.rados.rados.0.smithi033.stdout:3375: writing smithi03385152-28 from 2891776 to 3653632 tid 1 2024-09-21T10:02:08.575 INFO:tasks.rados.rados.0.smithi033.stdout:3376: delete oid 3 current snap is 308 2024-09-21T10:02:08.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1588 (ObjNum 943 snap 276 seq_num 943) dirty exists 2024-09-21T10:02:08.575 INFO:tasks.rados.rados.0.smithi033.stdout:3372: finishing rollback tid 0 to smithi03385152-17 2024-09-21T10:02:08.576 INFO:tasks.rados.rados.0.smithi033.stdout:3372: finishing rollback tid 1 to smithi03385152-17 2024-09-21T10:02:08.576 INFO:tasks.rados.rados.0.smithi033.stdout:3372: finishing rollback tid 2 to smithi03385152-17 2024-09-21T10:02:08.576 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2362 (ObjNum 947 snap 279 seq_num 947) dirty exists 2024-09-21T10:02:08.576 INFO:tasks.rados.rados.0.smithi033.stdout:3374: expect (ObjNum 982 snap 286 seq_num 982) 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:3375: finishing write tid 1 to smithi03385152-28 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:3375: finishing write tid 2 to smithi03385152-28 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:3371: done (5 left) 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:3372: done (4 left) 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:3374: done (3 left) 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:3376: done (2 left) 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 308 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:3375: finishing write tid 3 to smithi03385152-28 2024-09-21T10:02:08.639 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1830 (ObjNum 1048 snap 308 seq_num 1048) dirty exists 2024-09-21T10:02:08.640 INFO:tasks.rados.rados.0.smithi033.stdout:3375: left oid 28 (ObjNum 1048 snap 308 seq_num 1048) 2024-09-21T10:02:08.640 INFO:tasks.rados.rados.0.smithi033.stdout:3373: finishing copy_from racing read to smithi03385152-9 2024-09-21T10:02:08.640 INFO:tasks.rados.rados.0.smithi033.stdout:3377: seq_num 1050 ranges {475136=794624} 2024-09-21T10:02:08.646 INFO:tasks.rados.rados.0.smithi033.stdout:3377: writing smithi03385152-16 from 475136 to 1269760 tid 1 2024-09-21T10:02:08.646 INFO:tasks.rados.rados.0.smithi033.stdout:3375: done (2 left) 2024-09-21T10:02:08.646 INFO:tasks.rados.rados.0.smithi033.stdout:3378: snap_create 2024-09-21T10:02:08.646 INFO:tasks.rados.rados.0.smithi033.stdout:3373: finishing copy_from to smithi03385152-9 2024-09-21T10:02:08.646 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 1521 (ObjNum 1001 snap 292 seq_num 1001) dirty exists 2024-09-21T10:02:08.665 INFO:tasks.rados.rados.0.smithi033.stdout:3377: finishing write tid 1 to smithi03385152-16 2024-09-21T10:02:08.665 INFO:tasks.rados.rados.0.smithi033.stdout:3377: finishing write tid 2 to smithi03385152-16 2024-09-21T10:02:08.666 INFO:tasks.rados.rados.0.smithi033.stdout:3377: finishing write tid 3 to smithi03385152-16 2024-09-21T10:02:08.666 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2433 (ObjNum 1050 snap 308 seq_num 1050) dirty exists 2024-09-21T10:02:08.666 INFO:tasks.rados.rados.0.smithi033.stdout:3377: left oid 16 (ObjNum 1050 snap 308 seq_num 1050) 2024-09-21T10:02:09.617 INFO:tasks.rados.rados.0.smithi033.stdout:3373: done (2 left) 2024-09-21T10:02:09.617 INFO:tasks.rados.rados.0.smithi033.stdout:3377: done (1 left) 2024-09-21T10:02:09.617 INFO:tasks.rados.rados.0.smithi033.stdout:3378: done (0 left) 2024-09-21T10:02:09.617 INFO:tasks.rados.rados.0.smithi033.stdout:3379: copy_from oid 40 from oid 20 current snap is 309 2024-09-21T10:02:09.617 INFO:tasks.rados.rados.0.smithi033.stdout:3380: rmattr oid 46 current snap is 309 2024-09-21T10:02:09.617 INFO:tasks.rados.rados.0.smithi033.stdout:3380: done (1 left) 2024-09-21T10:02:09.617 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 309 2024-09-21T10:02:09.617 INFO:tasks.rados.rados.0.smithi033.stdout:3381: seq_num 1051 ranges {9363456=409600,9773056=401408} 2024-09-21T10:02:09.671 INFO:tasks.rados.rados.0.smithi033.stdout:3381: writing smithi03385152-48 from 9363456 to 9773056 tid 1 2024-09-21T10:02:09.673 INFO:tasks.rados.rados.0.smithi033.stdout:3381: writing smithi03385152-48 from 9773056 to 10174464 tid 2 2024-09-21T10:02:09.673 INFO:tasks.rados.rados.0.smithi033.stdout:3382: snap_create 2024-09-21T10:02:09.673 INFO:tasks.rados.rados.0.smithi033.stdout:3379: finishing copy_from racing read to smithi03385152-40 2024-09-21T10:02:09.673 INFO:tasks.rados.rados.0.smithi033.stdout:3379: finishing copy_from to smithi03385152-40 2024-09-21T10:02:09.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 1633 (ObjNum 1030 snap 303 seq_num 1030) dirty exists 2024-09-21T10:02:09.681 INFO:tasks.rados.rados.0.smithi033.stdout:3381: finishing write tid 1 to smithi03385152-48 2024-09-21T10:02:09.681 INFO:tasks.rados.rados.0.smithi033.stdout:3381: finishing write tid 2 to smithi03385152-48 2024-09-21T10:02:09.681 INFO:tasks.rados.rados.0.smithi033.stdout:3381: finishing write tid 3 to smithi03385152-48 2024-09-21T10:02:09.682 INFO:tasks.rados.rados.0.smithi033.stdout:3381: finishing write tid 4 to smithi03385152-48 2024-09-21T10:02:09.682 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 2394 (ObjNum 1051 snap 309 seq_num 1051) dirty exists 2024-09-21T10:02:09.682 INFO:tasks.rados.rados.0.smithi033.stdout:3381: left oid 48 (ObjNum 1051 snap 309 seq_num 1051) 2024-09-21T10:02:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:09 smithi033 ceph-mon[96568]: pgmap v674: 121 pgs: 121 active+clean; 152 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.4 MiB/s wr, 12 op/s 2024-09-21T10:02:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:09 smithi033 ceph-mon[96568]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T10:02:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:09 smithi033 ceph-mon[98546]: pgmap v674: 121 pgs: 121 active+clean; 152 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.4 MiB/s wr, 12 op/s 2024-09-21T10:02:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:09 smithi033 ceph-mon[98546]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T10:02:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:09 smithi105 ceph-mon[86649]: pgmap v674: 121 pgs: 121 active+clean; 152 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.4 MiB/s wr, 12 op/s 2024-09-21T10:02:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:09 smithi105 ceph-mon[86649]: osdmap e695: 8 total, 8 up, 8 in 2024-09-21T10:02:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:02:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:02:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:10.625 INFO:tasks.rados.rados.0.smithi033.stdout:3379: done (2 left) 2024-09-21T10:02:10.625 INFO:tasks.rados.rados.0.smithi033.stdout:3381: done (1 left) 2024-09-21T10:02:10.625 INFO:tasks.rados.rados.0.smithi033.stdout:3382: done (0 left) 2024-09-21T10:02:10.626 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 310 2024-09-21T10:02:10.626 INFO:tasks.rados.rados.0.smithi033.stdout:3383: seq_num 1052 ranges {7405568=425984,7831552=401408} 2024-09-21T10:02:10.668 INFO:tasks.rados.rados.0.smithi033.stdout:3383: writing smithi03385152-39 from 7405568 to 7831552 tid 1 2024-09-21T10:02:10.670 INFO:tasks.rados.rados.0.smithi033.stdout:3383: writing smithi03385152-39 from 7831552 to 8232960 tid 2 2024-09-21T10:02:10.670 INFO:tasks.rados.rados.0.smithi033.stdout:3384: read oid 17 snap -1 2024-09-21T10:02:10.671 INFO:tasks.rados.rados.0.smithi033.stdout:3384: expect (ObjNum 947 snap 279 seq_num 947) 2024-09-21T10:02:10.671 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 310 2024-09-21T10:02:10.671 INFO:tasks.rados.rados.0.smithi033.stdout:3385: seq_num 1053 ranges {6561792=442368,7004160=401408} 2024-09-21T10:02:10.709 INFO:tasks.rados.rados.0.smithi033.stdout:3385: writing smithi03385152-42 from 6561792 to 7004160 tid 1 2024-09-21T10:02:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3385: writing smithi03385152-42 from 7004160 to 7405568 tid 2 2024-09-21T10:02:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3386: read oid 11 snap -1 2024-09-21T10:02:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3386: expect (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:02:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3387: rollback oid 7 current snap is 310 2024-09-21T10:02:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 7 to 307 2024-09-21T10:02:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3388: snap_create 2024-09-21T10:02:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3383: finishing write tid 1 to smithi03385152-39 2024-09-21T10:02:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3383: finishing write tid 2 to smithi03385152-39 2024-09-21T10:02:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3383: finishing write tid 3 to smithi03385152-39 2024-09-21T10:02:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3383: finishing write tid 4 to smithi03385152-39 2024-09-21T10:02:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 2391 (ObjNum 1052 snap 310 seq_num 1052) dirty exists 2024-09-21T10:02:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3383: left oid 39 (ObjNum 1052 snap 310 seq_num 1052) 2024-09-21T10:02:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3384: expect (ObjNum 947 snap 279 seq_num 947) 2024-09-21T10:02:10.758 INFO:tasks.rados.rados.0.smithi033.stdout:3385: finishing write tid 1 to smithi03385152-42 2024-09-21T10:02:10.758 INFO:tasks.rados.rados.0.smithi033.stdout:3387: finishing rollback tid 0 to smithi03385152-7 2024-09-21T10:02:10.758 INFO:tasks.rados.rados.0.smithi033.stdout:3385: finishing write tid 2 to smithi03385152-42 2024-09-21T10:02:10.759 INFO:tasks.rados.rados.0.smithi033.stdout:3387: finishing rollback tid 1 to smithi03385152-7 2024-09-21T10:02:10.759 INFO:tasks.rados.rados.0.smithi033.stdout:3385: finishing write tid 3 to smithi03385152-42 2024-09-21T10:02:10.759 INFO:tasks.rados.rados.0.smithi033.stdout:3387: finishing rollback tid 2 to smithi03385152-7 2024-09-21T10:02:10.759 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2437 (ObjNum 1007 snap 296 seq_num 1007) dirty exists 2024-09-21T10:02:10.759 INFO:tasks.rados.rados.0.smithi033.stdout:3385: finishing write tid 4 to smithi03385152-42 2024-09-21T10:02:10.759 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 2381 (ObjNum 1053 snap 310 seq_num 1053) dirty exists 2024-09-21T10:02:10.759 INFO:tasks.rados.rados.0.smithi033.stdout:3385: left oid 42 (ObjNum 1053 snap 310 seq_num 1053) 2024-09-21T10:02:10.759 INFO:tasks.rados.rados.0.smithi033.stdout:3386: expect (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:02:10.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:10 smithi033 ceph-mon[98546]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T10:02:10.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:10 smithi033 ceph-mon[96568]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T10:02:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:10 smithi105 ceph-mon[86649]: osdmap e696: 8 total, 8 up, 8 in 2024-09-21T10:02:11.633 INFO:tasks.rados.rados.0.smithi033.stdout:3383: done (5 left) 2024-09-21T10:02:11.633 INFO:tasks.rados.rados.0.smithi033.stdout:3384: done (4 left) 2024-09-21T10:02:11.633 INFO:tasks.rados.rados.0.smithi033.stdout:3385: done (3 left) 2024-09-21T10:02:11.633 INFO:tasks.rados.rados.0.smithi033.stdout:3386: done (2 left) 2024-09-21T10:02:11.633 INFO:tasks.rados.rados.0.smithi033.stdout:3387: done (1 left) 2024-09-21T10:02:11.633 INFO:tasks.rados.rados.0.smithi033.stdout:3388: done (0 left) 2024-09-21T10:02:11.633 INFO:tasks.rados.rados.0.smithi033.stdout:3389: copy_from oid 35 from oid 31 current snap is 311 2024-09-21T10:02:11.633 INFO:tasks.rados.rados.0.smithi033.stdout:3390: snap_remove snap 304 2024-09-21T10:02:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:11 smithi033 ceph-mon[98546]: pgmap v677: 121 pgs: 121 active+clean; 180 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.1 MiB/s wr, 17 op/s 2024-09-21T10:02:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:11 smithi033 ceph-mon[98546]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T10:02:11.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:02:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:02:11] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-09-21T10:02:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:11 smithi033 ceph-mon[96568]: pgmap v677: 121 pgs: 121 active+clean; 180 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.1 MiB/s wr, 17 op/s 2024-09-21T10:02:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:11 smithi033 ceph-mon[96568]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T10:02:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:11 smithi105 ceph-mon[86649]: pgmap v677: 121 pgs: 121 active+clean; 180 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.1 MiB/s wr, 17 op/s 2024-09-21T10:02:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:11 smithi105 ceph-mon[86649]: osdmap e697: 8 total, 8 up, 8 in 2024-09-21T10:02:12.636 INFO:tasks.rados.rados.0.smithi033.stdout:3389: finishing copy_from racing read to smithi03385152-35 2024-09-21T10:02:12.637 INFO:tasks.rados.rados.0.smithi033.stdout:3389: finishing copy_from to smithi03385152-35 2024-09-21T10:02:12.637 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2427 (ObjNum 1040 snap 307 seq_num 1040) dirty exists 2024-09-21T10:02:12.637 INFO:tasks.rados.rados.0.smithi033.stdout:3389: done (1 left) 2024-09-21T10:02:12.637 INFO:tasks.rados.rados.0.smithi033.stdout:3390: done (0 left) 2024-09-21T10:02:12.637 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 311 2024-09-21T10:02:12.637 INFO:tasks.rados.rados.0.smithi033.stdout:3391: seq_num 1054 ranges {2023424=458752,2482176=401408} 2024-09-21T10:02:12.651 INFO:tasks.rados.rados.0.smithi033.stdout:3391: writing smithi03385152-41 from 2023424 to 2482176 tid 1 2024-09-21T10:02:12.654 INFO:tasks.rados.rados.0.smithi033.stdout:3391: writing smithi03385152-41 from 2482176 to 2883584 tid 2 2024-09-21T10:02:12.654 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 311 2024-09-21T10:02:12.655 INFO:tasks.rados.rados.0.smithi033.stdout:3392: seq_num 1055 ranges {0=475136,475136=401408} 2024-09-21T10:02:12.657 INFO:tasks.rados.rados.0.smithi033.stdout:3392: writing smithi03385152-2 from 0 to 475136 tid 1 2024-09-21T10:02:12.660 INFO:tasks.rados.rados.0.smithi033.stdout:3392: writing smithi03385152-2 from 475136 to 876544 tid 2 2024-09-21T10:02:12.660 INFO:tasks.rados.rados.0.smithi033.stdout:3393: setattr oid 18 current snap is 311 2024-09-21T10:02:12.660 INFO:tasks.rados.rados.0.smithi033.stdout:3391: finishing write tid 1 to smithi03385152-41 2024-09-21T10:02:12.661 INFO:tasks.rados.rados.0.smithi033.stdout:3391: finishing write tid 2 to smithi03385152-41 2024-09-21T10:02:12.661 INFO:tasks.rados.rados.0.smithi033.stdout:3391: finishing write tid 3 to smithi03385152-41 2024-09-21T10:02:12.663 INFO:tasks.rados.rados.0.smithi033.stdout:3394: snap_remove snap 309 2024-09-21T10:02:12.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:12 smithi033 ceph-mon[98546]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T10:02:12.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:12 smithi033 ceph-mon[96568]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T10:02:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:12 smithi105 ceph-mon[86649]: osdmap e698: 8 total, 8 up, 8 in 2024-09-21T10:02:13.561 INFO:tasks.rados.rados.0.smithi033.stdout:3394: done (3 left) 2024-09-21T10:02:13.561 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 311 2024-09-21T10:02:13.561 INFO:tasks.rados.rados.0.smithi033.stdout:3395: seq_num 1056 ranges {10321920=491520,10813440=401408} 2024-09-21T10:02:13.620 INFO:tasks.rados.rados.0.smithi033.stdout:3395: writing smithi03385152-31 from 10321920 to 10813440 tid 1 2024-09-21T10:02:13.623 INFO:tasks.rados.rados.0.smithi033.stdout:3395: writing smithi03385152-31 from 10813440 to 11214848 tid 2 2024-09-21T10:02:13.623 INFO:tasks.rados.rados.0.smithi033.stdout:3396: read oid 4 snap -1 2024-09-21T10:02:13.623 INFO:tasks.rados.rados.0.smithi033.stdout:3396: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:13.623 INFO:tasks.rados.rados.0.smithi033.stdout:3397: setattr oid 13 current snap is 311 2024-09-21T10:02:13.623 INFO:tasks.rados.rados.0.smithi033.stdout:3391: finishing write tid 4 to smithi03385152-41 2024-09-21T10:02:13.623 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 2384 (ObjNum 1054 snap 311 seq_num 1054) dirty exists 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:3391: left oid 41 (ObjNum 1054 snap 311 seq_num 1054) 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:3392: finishing write tid 1 to smithi03385152-2 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:3392: finishing write tid 2 to smithi03385152-2 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:3392: finishing write tid 3 to smithi03385152-2 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1822 (ObjNum 1044 snap 307 seq_num 1044) dirty exists 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:3392: finishing write tid 4 to smithi03385152-2 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 1803 (ObjNum 1055 snap 311 seq_num 1055) dirty exists 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:3392: left oid 2 (ObjNum 1055 snap 311 seq_num 1055) 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:3391: done (5 left) 2024-09-21T10:02:13.624 INFO:tasks.rados.rados.0.smithi033.stdout:3392: done (4 left) 2024-09-21T10:02:13.625 INFO:tasks.rados.rados.0.smithi033.stdout:3393: done (3 left) 2024-09-21T10:02:13.625 INFO:tasks.rados.rados.0.smithi033.stdout:3398: setattr oid 49 current snap is 311 2024-09-21T10:02:13.626 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 311 2024-09-21T10:02:13.627 INFO:tasks.rados.rados.0.smithi033.stdout:3399: seq_num 1057 ranges {7405568=507904,7913472=401408} 2024-09-21T10:02:13.670 INFO:tasks.rados.rados.0.smithi033.stdout:3399: writing smithi03385152-42 from 7405568 to 7913472 tid 1 2024-09-21T10:02:13.673 INFO:tasks.rados.rados.0.smithi033.stdout:3399: writing smithi03385152-42 from 7913472 to 8314880 tid 2 2024-09-21T10:02:13.673 INFO:tasks.rados.rados.0.smithi033.stdout:3400: delete oid 46 current snap is 311 2024-09-21T10:02:13.673 INFO:tasks.rados.rados.0.smithi033.stdout:3395: finishing write tid 1 to smithi03385152-31 2024-09-21T10:02:13.673 INFO:tasks.rados.rados.0.smithi033.stdout:3395: finishing write tid 2 to smithi03385152-31 2024-09-21T10:02:13.673 INFO:tasks.rados.rados.0.smithi033.stdout:3395: finishing write tid 3 to smithi03385152-31 2024-09-21T10:02:13.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 2363 (ObjNum 987 snap 288 seq_num 987) dirty exists 2024-09-21T10:02:13.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2454 (ObjNum 1020 snap 302 seq_num 1020) dirty exists 2024-09-21T10:02:13.673 INFO:tasks.rados.rados.0.smithi033.stdout:3395: finishing write tid 4 to smithi03385152-31 2024-09-21T10:02:13.674 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2452 (ObjNum 1056 snap 311 seq_num 1056) dirty exists 2024-09-21T10:02:13.674 INFO:tasks.rados.rados.0.smithi033.stdout:3395: left oid 31 (ObjNum 1056 snap 311 seq_num 1056) 2024-09-21T10:02:13.674 INFO:tasks.rados.rados.0.smithi033.stdout:3396: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3399: finishing write tid 1 to smithi03385152-42 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3399: finishing write tid 2 to smithi03385152-42 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3395: done (5 left) 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3396: done (4 left) 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3397: done (3 left) 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3398: done (2 left) 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3400: done (1 left) 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3401: rollback oid 27 current snap is 311 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 27 to 306 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3399: finishing write tid 3 to smithi03385152-42 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3399: finishing write tid 4 to smithi03385152-42 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 2387 (ObjNum 1057 snap 311 seq_num 1057) dirty exists 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3399: left oid 42 (ObjNum 1057 snap 311 seq_num 1057) 2024-09-21T10:02:13.728 INFO:tasks.rados.rados.0.smithi033.stdout:3399: done (1 left) 2024-09-21T10:02:13.729 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 311 2024-09-21T10:02:13.729 INFO:tasks.rados.rados.0.smithi033.stdout:3402: seq_num 1059 ranges {1458176=540672,1998848=401408} 2024-09-21T10:02:13.738 INFO:tasks.rados.rados.0.smithi033.stdout:3402: writing smithi03385152-32 from 1458176 to 1998848 tid 1 2024-09-21T10:02:13.741 INFO:tasks.rados.rados.0.smithi033.stdout:3402: writing smithi03385152-32 from 1998848 to 2400256 tid 2 2024-09-21T10:02:13.741 INFO:tasks.rados.rados.0.smithi033.stdout:3403: read oid 46 snap -1 2024-09-21T10:02:13.741 INFO:tasks.rados.rados.0.smithi033.stdout:3403: expect deleted 2024-09-21T10:02:13.741 INFO:tasks.rados.rados.0.smithi033.stdout:3404: snap_remove snap 307 2024-09-21T10:02:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:13 smithi033 ceph-mon[96568]: pgmap v680: 121 pgs: 121 active+clean; 185 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 9.0 MiB/s wr, 20 op/s 2024-09-21T10:02:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:13 smithi033 ceph-mon[96568]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T10:02:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:13 smithi033 ceph-mon[96568]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T10:02:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:13 smithi033 ceph-mon[98546]: pgmap v680: 121 pgs: 121 active+clean; 185 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 9.0 MiB/s wr, 20 op/s 2024-09-21T10:02:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:13 smithi033 ceph-mon[98546]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T10:02:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:13 smithi033 ceph-mon[98546]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T10:02:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:13 smithi105 ceph-mon[86649]: pgmap v680: 121 pgs: 121 active+clean; 185 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 9.0 MiB/s wr, 20 op/s 2024-09-21T10:02:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:13 smithi105 ceph-mon[86649]: osdmap e699: 8 total, 8 up, 8 in 2024-09-21T10:02:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:13 smithi105 ceph-mon[86649]: osdmap e700: 8 total, 8 up, 8 in 2024-09-21T10:02:14.644 INFO:tasks.rados.rados.0.smithi033.stdout:3401: finishing rollback tid 1 to smithi03385152-27 2024-09-21T10:02:14.644 INFO:tasks.rados.rados.0.smithi033.stdout:3401: finishing rollback tid 2 to smithi03385152-27 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2272 (ObjNum 1033 snap 304 seq_num 1033) dirty exists 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3402: finishing write tid 1 to smithi03385152-32 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3402: finishing write tid 2 to smithi03385152-32 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3402: finishing write tid 3 to smithi03385152-32 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3402: finishing write tid 4 to smithi03385152-32 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1591 (ObjNum 1059 snap 311 seq_num 1059) dirty exists 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3402: left oid 32 (ObjNum 1059 snap 311 seq_num 1059) 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3401: done (3 left) 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3402: done (2 left) 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3403: done (1 left) 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3404: done (0 left) 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3405: read oid 11 snap -1 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3405: expect (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:02:14.645 INFO:tasks.rados.rados.0.smithi033.stdout:3406: snap_remove snap 308 2024-09-21T10:02:14.780 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:02:14 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=infra.usagestats t=2024-09-21T10:02:14.457465809Z level=info msg="Usage stats are ready to report" 2024-09-21T10:02:15.645 INFO:tasks.rados.rados.0.smithi033.stdout:3406: done (1 left) 2024-09-21T10:02:15.645 INFO:tasks.rados.rados.0.smithi033.stdout:3407: read oid 29 snap -1 2024-09-21T10:02:15.645 INFO:tasks.rados.rados.0.smithi033.stdout:3407: expect (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:02:15.645 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 311 2024-09-21T10:02:15.645 INFO:tasks.rados.rados.0.smithi033.stdout:3408: seq_num 1060 ranges {10321920=557056,10878976=401408} 2024-09-21T10:02:15.705 INFO:tasks.rados.rados.0.smithi033.stdout:3408: writing smithi03385152-35 from 10321920 to 10878976 tid 1 2024-09-21T10:02:15.708 INFO:tasks.rados.rados.0.smithi033.stdout:3408: writing smithi03385152-35 from 10878976 to 11280384 tid 2 2024-09-21T10:02:15.708 INFO:tasks.rados.rados.0.smithi033.stdout:3409: copy_from oid 17 from oid 4 current snap is 311 2024-09-21T10:02:15.708 INFO:tasks.rados.rados.0.smithi033.stdout:3405: expect (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:02:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:15 smithi033 ceph-mon[98546]: pgmap v683: 121 pgs: 121 active+clean; 191 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.4 MiB/s wr, 21 op/s 2024-09-21T10:02:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:15 smithi033 ceph-mon[98546]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T10:02:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:15 smithi033 ceph-mon[96568]: pgmap v683: 121 pgs: 121 active+clean; 191 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.4 MiB/s wr, 21 op/s 2024-09-21T10:02:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:15 smithi033 ceph-mon[96568]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T10:02:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:15 smithi105 ceph-mon[86649]: pgmap v683: 121 pgs: 121 active+clean; 191 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 4.4 MiB/s wr, 21 op/s 2024-09-21T10:02:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:15 smithi105 ceph-mon[86649]: osdmap e701: 8 total, 8 up, 8 in 2024-09-21T10:02:16.173 INFO:tasks.rados.rados.0.smithi033.stdout:3407: expect (ObjNum 997 snap 292 seq_num 997) 2024-09-21T10:02:16.420 INFO:tasks.rados.rados.0.smithi033.stdout:3408: finishing write tid 1 to smithi03385152-35 2024-09-21T10:02:16.420 INFO:tasks.rados.rados.0.smithi033.stdout:3408: finishing write tid 2 to smithi03385152-35 2024-09-21T10:02:16.420 INFO:tasks.rados.rados.0.smithi033.stdout:3408: finishing write tid 3 to smithi03385152-35 2024-09-21T10:02:16.420 INFO:tasks.rados.rados.0.smithi033.stdout:3408: finishing write tid 4 to smithi03385152-35 2024-09-21T10:02:16.420 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2471 (ObjNum 1060 snap 311 seq_num 1060) dirty exists 2024-09-21T10:02:16.420 INFO:tasks.rados.rados.0.smithi033.stdout:3408: left oid 35 (ObjNum 1060 snap 311 seq_num 1060) 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:3409: finishing copy_from racing read to smithi03385152-17 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:3405: done (3 left) 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:3407: done (2 left) 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:3408: done (1 left) 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:3410: read oid 35 snap -1 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:3410: expect (ObjNum 1060 snap 311 seq_num 1060) 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 311 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:3409: finishing copy_from to smithi03385152-17 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:16.421 INFO:tasks.rados.rados.0.smithi033.stdout:3411: seq_num 1061 ranges {4022272=573440,4595712=401408} 2024-09-21T10:02:16.445 INFO:tasks.rados.rados.0.smithi033.stdout:3411: writing smithi03385152-30 from 4022272 to 4595712 tid 1 2024-09-21T10:02:16.447 INFO:tasks.rados.rados.0.smithi033.stdout:3411: writing smithi03385152-30 from 4595712 to 4997120 tid 2 2024-09-21T10:02:16.447 INFO:tasks.rados.rados.0.smithi033.stdout:3409: done (2 left) 2024-09-21T10:02:16.447 INFO:tasks.rados.rados.0.smithi033.stdout:3412: read oid 15 snap -1 2024-09-21T10:02:16.447 INFO:tasks.rados.rados.0.smithi033.stdout:3412: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:16.447 INFO:tasks.rados.rados.0.smithi033.stdout:3413: snap_create 2024-09-21T10:02:16.464 INFO:tasks.rados.rados.0.smithi033.stdout:3412: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:16.518 INFO:tasks.rados.rados.0.smithi033.stdout:3410: expect (ObjNum 1060 snap 311 seq_num 1060) 2024-09-21T10:02:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:16 smithi033 ceph-mon[96568]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T10:02:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:16 smithi033 ceph-mon[98546]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T10:02:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:16 smithi105 ceph-mon[86649]: osdmap e702: 8 total, 8 up, 8 in 2024-09-21T10:02:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:17.835 INFO:tasks.rados.rados.0.smithi033.stdout:3411: finishing write tid 1 to smithi03385152-30 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3411: finishing write tid 2 to smithi03385152-30 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3411: finishing write tid 3 to smithi03385152-30 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3411: finishing write tid 4 to smithi03385152-30 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2475 (ObjNum 1061 snap 311 seq_num 1061) dirty exists 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3411: left oid 30 (ObjNum 1061 snap 311 seq_num 1061) 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3410: done (3 left) 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3411: done (2 left) 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3412: done (1 left) 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3413: done (0 left) 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 312 2024-09-21T10:02:17.836 INFO:tasks.rados.rados.0.smithi033.stdout:3414: seq_num 1062 ranges {8314880=589824,8904704=401408} 2024-09-21T10:02:17.884 INFO:tasks.rados.rados.0.smithi033.stdout:3414: writing smithi03385152-42 from 8314880 to 8904704 tid 1 2024-09-21T10:02:17.886 INFO:tasks.rados.rados.0.smithi033.stdout:3414: writing smithi03385152-42 from 8904704 to 9306112 tid 2 2024-09-21T10:02:17.886 INFO:tasks.rados.rados.0.smithi033.stdout:3415: snap_remove snap 305 2024-09-21T10:02:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:17 smithi033 ceph-mon[98546]: pgmap v686: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 6.0 MiB/s wr, 20 op/s 2024-09-21T10:02:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:17 smithi033 ceph-mon[98546]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T10:02:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:17 smithi033 ceph-mon[96568]: pgmap v686: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 6.0 MiB/s wr, 20 op/s 2024-09-21T10:02:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:17 smithi033 ceph-mon[96568]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T10:02:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:17 smithi105 ceph-mon[86649]: pgmap v686: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 6.0 MiB/s wr, 20 op/s 2024-09-21T10:02:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:17 smithi105 ceph-mon[86649]: osdmap e703: 8 total, 8 up, 8 in 2024-09-21T10:02:18.664 INFO:tasks.rados.rados.0.smithi033.stdout:3414: finishing write tid 1 to smithi03385152-42 2024-09-21T10:02:18.664 INFO:tasks.rados.rados.0.smithi033.stdout:3414: finishing write tid 2 to smithi03385152-42 2024-09-21T10:02:18.664 INFO:tasks.rados.rados.0.smithi033.stdout:3414: finishing write tid 3 to smithi03385152-42 2024-09-21T10:02:18.664 INFO:tasks.rados.rados.0.smithi033.stdout:3414: finishing write tid 4 to smithi03385152-42 2024-09-21T10:02:18.664 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 2390 (ObjNum 1062 snap 312 seq_num 1062) dirty exists 2024-09-21T10:02:18.665 INFO:tasks.rados.rados.0.smithi033.stdout:3414: left oid 42 (ObjNum 1062 snap 312 seq_num 1062) 2024-09-21T10:02:18.665 INFO:tasks.rados.rados.0.smithi033.stdout:3414: done (1 left) 2024-09-21T10:02:18.665 INFO:tasks.rados.rados.0.smithi033.stdout:3415: done (0 left) 2024-09-21T10:02:18.665 INFO:tasks.rados.rados.0.smithi033.stdout:3416: rollback oid 48 current snap is 312 2024-09-21T10:02:18.665 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 48 to 310 2024-09-21T10:02:18.665 INFO:tasks.rados.rados.0.smithi033.stdout:3417: copy_from oid 21 from oid 16 current snap is 312 2024-09-21T10:02:18.665 INFO:tasks.rados.rados.0.smithi033.stdout:3418: snap_remove snap 306 2024-09-21T10:02:19.665 INFO:tasks.rados.rados.0.smithi033.stdout:3418: done (2 left) 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:3419: read oid 30 snap 310 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:3419: expect (ObjNum 1047 snap 307 seq_num 1047) 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:3420: snap_create 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:3416: finishing rollback tid 0 to smithi03385152-48 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:3416: finishing rollback tid 1 to smithi03385152-48 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:3416: finishing rollback tid 2 to smithi03385152-48 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 2398 (ObjNum 1051 snap 309 seq_num 1051) dirty exists 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:3417: finishing copy_from racing read to smithi03385152-21 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:3417: finishing copy_from to smithi03385152-21 2024-09-21T10:02:19.666 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 2433 (ObjNum 1050 snap 308 seq_num 1050) dirty exists 2024-09-21T10:02:19.690 INFO:tasks.rados.rados.0.smithi033.stdout:3419: expect (ObjNum 1047 snap 307 seq_num 1047) 2024-09-21T10:02:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:19 smithi033 ceph-mon[96568]: pgmap v688: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 5.0 MiB/s wr, 21 op/s 2024-09-21T10:02:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:19 smithi033 ceph-mon[96568]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T10:02:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:19 smithi033 ceph-mon[98546]: pgmap v688: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 5.0 MiB/s wr, 21 op/s 2024-09-21T10:02:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:19 smithi033 ceph-mon[98546]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T10:02:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:19 smithi105 ceph-mon[86649]: pgmap v688: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.6 MiB/s rd, 5.0 MiB/s wr, 21 op/s 2024-09-21T10:02:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:19 smithi105 ceph-mon[86649]: osdmap e704: 8 total, 8 up, 8 in 2024-09-21T10:02:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:02:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:02:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:20.673 INFO:tasks.rados.rados.0.smithi033.stdout:3416: done (3 left) 2024-09-21T10:02:20.673 INFO:tasks.rados.rados.0.smithi033.stdout:3417: done (2 left) 2024-09-21T10:02:20.673 INFO:tasks.rados.rados.0.smithi033.stdout:3419: done (1 left) 2024-09-21T10:02:20.673 INFO:tasks.rados.rados.0.smithi033.stdout:3420: done (0 left) 2024-09-21T10:02:20.673 INFO:tasks.rados.rados.0.smithi033.stdout:3421: copy_from oid 49 from oid 13 current snap is 313 2024-09-21T10:02:20.673 INFO:tasks.rados.rados.0.smithi033.stdout:3422: copy_from oid 46 from oid 17 current snap is 313 2024-09-21T10:02:20.673 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 313 2024-09-21T10:02:20.673 INFO:tasks.rados.rados.0.smithi033.stdout:3423: seq_num 1063 ranges {6561792=606208,7168000=401408} 2024-09-21T10:02:20.711 INFO:tasks.rados.rados.0.smithi033.stdout:3423: writing smithi03385152-25 from 6561792 to 7168000 tid 1 2024-09-21T10:02:20.714 INFO:tasks.rados.rados.0.smithi033.stdout:3423: writing smithi03385152-25 from 7168000 to 7569408 tid 2 2024-09-21T10:02:20.714 INFO:tasks.rados.rados.0.smithi033.stdout:3424: read oid 11 snap -1 2024-09-21T10:02:20.714 INFO:tasks.rados.rados.0.smithi033.stdout:3424: expect (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:02:20.714 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 313 2024-09-21T10:02:20.714 INFO:tasks.rados.rados.0.smithi033.stdout:3425: seq_num 1064 ranges {2629632=622592,3252224=401408} 2024-09-21T10:02:20.732 INFO:tasks.rados.rados.0.smithi033.stdout:3425: writing smithi03385152-12 from 2629632 to 3252224 tid 1 2024-09-21T10:02:20.734 INFO:tasks.rados.rados.0.smithi033.stdout:3425: writing smithi03385152-12 from 3252224 to 3653632 tid 2 2024-09-21T10:02:20.734 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 313 2024-09-21T10:02:20.734 INFO:tasks.rados.rados.0.smithi033.stdout:3422: finishing copy_from racing read to smithi03385152-46 2024-09-21T10:02:20.735 INFO:tasks.rados.rados.0.smithi033.stdout:3426: seq_num 1065 ranges {2400256=638976,3039232=401408} 2024-09-21T10:02:20.750 INFO:tasks.rados.rados.0.smithi033.stdout:3426: writing smithi03385152-32 from 2400256 to 3039232 tid 1 2024-09-21T10:02:20.754 INFO:tasks.rados.rados.0.smithi033.stdout:3426: writing smithi03385152-32 from 3039232 to 3440640 tid 2 2024-09-21T10:02:20.754 INFO:tasks.rados.rados.0.smithi033.stdout:3427: rollback oid 41 current snap is 313 2024-09-21T10:02:20.754 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 41 to 311 2024-09-21T10:02:20.754 INFO:tasks.rados.rados.0.smithi033.stdout:3422: finishing copy_from to smithi03385152-46 2024-09-21T10:02:20.754 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:20.754 INFO:tasks.rados.rados.0.smithi033.stdout:3421: finishing copy_from racing read to smithi03385152-49 2024-09-21T10:02:20.754 INFO:tasks.rados.rados.0.smithi033.stdout:3421: finishing copy_from to smithi03385152-49 2024-09-21T10:02:20.754 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 2454 (ObjNum 1020 snap 302 seq_num 1020) dirty exists 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3423: finishing write tid 1 to smithi03385152-25 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3423: finishing write tid 2 to smithi03385152-25 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3423: finishing write tid 3 to smithi03385152-25 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3423: finishing write tid 4 to smithi03385152-25 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 1806 (ObjNum 1063 snap 313 seq_num 1063) dirty exists 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3423: left oid 25 (ObjNum 1063 snap 313 seq_num 1063) 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3421: done (6 left) 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3422: done (5 left) 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3423: done (4 left) 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3428: delete oid 36 current snap is 313 2024-09-21T10:02:20.755 INFO:tasks.rados.rados.0.smithi033.stdout:3424: expect (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:02:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:20 smithi033 ceph-mon[96568]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T10:02:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:20 smithi033 ceph-mon[98546]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T10:02:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:20 smithi105 ceph-mon[86649]: osdmap e705: 8 total, 8 up, 8 in 2024-09-21T10:02:21.218 INFO:tasks.rados.rados.0.smithi033.stdout:3425: finishing write tid 1 to smithi03385152-12 2024-09-21T10:02:21.218 INFO:tasks.rados.rados.0.smithi033.stdout:3425: finishing write tid 2 to smithi03385152-12 2024-09-21T10:02:21.219 INFO:tasks.rados.rados.0.smithi033.stdout:3424: done (4 left) 2024-09-21T10:02:21.219 INFO:tasks.rados.rados.0.smithi033.stdout:3428: done (3 left) 2024-09-21T10:02:21.219 INFO:tasks.rados.rados.0.smithi033.stdout:3429: read oid 23 snap -1 2024-09-21T10:02:21.219 INFO:tasks.rados.rados.0.smithi033.stdout:3429: expect deleted 2024-09-21T10:02:21.219 INFO:tasks.rados.rados.0.smithi033.stdout:3430: read oid 30 snap 311 2024-09-21T10:02:21.219 INFO:tasks.rados.rados.0.smithi033.stdout:3430: expect (ObjNum 1061 snap 311 seq_num 1061) 2024-09-21T10:02:21.219 INFO:tasks.rados.rados.0.smithi033.stdout:3431: snap_remove snap 312 2024-09-21T10:02:21.690 INFO:tasks.rados.rados.0.smithi033.stdout:3431: done (5 left) 2024-09-21T10:02:21.691 INFO:tasks.rados.rados.0.smithi033.stdout:3432: snap_remove snap 310 2024-09-21T10:02:21.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:02:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:02:21] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-09-21T10:02:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:21 smithi033 ceph-mon[98546]: pgmap v691: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 993 KiB/s wr, 8 op/s 2024-09-21T10:02:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:21 smithi033 ceph-mon[98546]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T10:02:21.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:21 smithi033 ceph-mon[96568]: pgmap v691: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 993 KiB/s wr, 8 op/s 2024-09-21T10:02:21.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:21 smithi033 ceph-mon[96568]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T10:02:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:21 smithi105 ceph-mon[86649]: pgmap v691: 121 pgs: 121 active+clean; 196 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 993 KiB/s wr, 8 op/s 2024-09-21T10:02:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:21 smithi105 ceph-mon[86649]: osdmap e706: 8 total, 8 up, 8 in 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3432: done (5 left) 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3433: setattr oid 17 current snap is 313 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3425: finishing write tid 3 to smithi03385152-12 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3425: finishing write tid 4 to smithi03385152-12 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2457 (ObjNum 1064 snap 313 seq_num 1064) dirty exists 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3425: left oid 12 (ObjNum 1064 snap 313 seq_num 1064) 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3426: finishing write tid 1 to smithi03385152-32 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3427: finishing rollback tid 0 to smithi03385152-41 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3426: finishing write tid 2 to smithi03385152-32 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3427: finishing rollback tid 1 to smithi03385152-41 2024-09-21T10:02:22.698 INFO:tasks.rados.rados.0.smithi033.stdout:3426: finishing write tid 3 to smithi03385152-32 2024-09-21T10:02:22.699 INFO:tasks.rados.rados.0.smithi033.stdout:3427: finishing rollback tid 2 to smithi03385152-41 2024-09-21T10:02:22.699 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 2393 (ObjNum 1054 snap 311 seq_num 1054) dirty exists 2024-09-21T10:02:22.699 INFO:tasks.rados.rados.0.smithi033.stdout:3426: finishing write tid 4 to smithi03385152-32 2024-09-21T10:02:22.699 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1594 (ObjNum 1065 snap 313 seq_num 1065) dirty exists 2024-09-21T10:02:22.699 INFO:tasks.rados.rados.0.smithi033.stdout:3426: left oid 32 (ObjNum 1065 snap 313 seq_num 1065) 2024-09-21T10:02:22.699 INFO:tasks.rados.rados.0.smithi033.stdout:3430: expect (ObjNum 1061 snap 311 seq_num 1061) 2024-09-21T10:02:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:22 smithi033 ceph-mon[98546]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T10:02:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:22 smithi033 ceph-mon[96568]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T10:02:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:22 smithi105 ceph-mon[86649]: osdmap e707: 8 total, 8 up, 8 in 2024-09-21T10:02:23.033 INFO:tasks.rados.rados.0.smithi033.stdout:3425: done (5 left) 2024-09-21T10:02:23.034 INFO:tasks.rados.rados.0.smithi033.stdout:3426: done (4 left) 2024-09-21T10:02:23.034 INFO:tasks.rados.rados.0.smithi033.stdout:3427: done (3 left) 2024-09-21T10:02:23.034 INFO:tasks.rados.rados.0.smithi033.stdout:3429: done (2 left) 2024-09-21T10:02:23.034 INFO:tasks.rados.rados.0.smithi033.stdout:3430: done (1 left) 2024-09-21T10:02:23.034 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 313 2024-09-21T10:02:23.034 INFO:tasks.rados.rados.0.smithi033.stdout:3434: seq_num 1067 ranges {11214848=679936,11894784=401408} 2024-09-21T10:02:23.098 INFO:tasks.rados.rados.0.smithi033.stdout:3434: writing smithi03385152-31 from 11214848 to 11894784 tid 1 2024-09-21T10:02:23.100 INFO:tasks.rados.rados.0.smithi033.stdout:3434: writing smithi03385152-31 from 11894784 to 12296192 tid 2 2024-09-21T10:02:23.101 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 313 2024-09-21T10:02:23.101 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2458 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:23.101 INFO:tasks.rados.rados.0.smithi033.stdout:3435: seq_num 1068 ranges {688128=696320,1384448=401408} 2024-09-21T10:02:23.108 INFO:tasks.rados.rados.0.smithi033.stdout:3435: writing smithi03385152-18 from 688128 to 1384448 tid 1 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:3435: writing smithi03385152-18 from 1384448 to 1785856 tid 2 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:3433: done (2 left) 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:3436: delete oid 23 current snap is 313 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:3434: finishing write tid 1 to smithi03385152-31 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:3434: finishing write tid 2 to smithi03385152-31 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:3434: finishing write tid 3 to smithi03385152-31 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:3434: finishing write tid 4 to smithi03385152-31 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2511 (ObjNum 1067 snap 313 seq_num 1067) dirty exists 2024-09-21T10:02:23.112 INFO:tasks.rados.rados.0.smithi033.stdout:3434: left oid 31 (ObjNum 1067 snap 313 seq_num 1067) 2024-09-21T10:02:23.113 INFO:tasks.rados.rados.0.smithi033.stdout:3434: done (2 left) 2024-09-21T10:02:23.113 INFO:tasks.rados.rados.0.smithi033.stdout:3436: done (1 left) 2024-09-21T10:02:23.113 INFO:tasks.rados.rados.0.smithi033.stdout:3437: snap_remove snap 311 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:3435: finishing write tid 1 to smithi03385152-18 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:3435: finishing write tid 2 to smithi03385152-18 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:3435: finishing write tid 3 to smithi03385152-18 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:3435: finishing write tid 4 to smithi03385152-18 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1825 (ObjNum 1068 snap 313 seq_num 1068) dirty exists 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:3435: left oid 18 (ObjNum 1068 snap 313 seq_num 1068) 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:3435: done (1 left) 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:3437: done (0 left) 2024-09-21T10:02:23.565 INFO:tasks.rados.rados.0.smithi033.stdout:3438: copy_from oid 6 from oid 30 current snap is 313 2024-09-21T10:02:23.566 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 313 2024-09-21T10:02:23.566 INFO:tasks.rados.rados.0.smithi033.stdout:3439: seq_num 1070 ranges {2072576=729088,2801664=401408} 2024-09-21T10:02:23.581 INFO:tasks.rados.rados.0.smithi033.stdout:3439: writing smithi03385152-22 from 2072576 to 2801664 tid 1 2024-09-21T10:02:23.583 INFO:tasks.rados.rados.0.smithi033.stdout:3439: writing smithi03385152-22 from 2801664 to 3203072 tid 2 2024-09-21T10:02:23.583 INFO:tasks.rados.rados.0.smithi033.stdout:3440: delete oid 26 current snap is 313 2024-09-21T10:02:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3440: done (2 left) 2024-09-21T10:02:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3441: rollback oid 48 current snap is 313 2024-09-21T10:02:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3441: done (2 left) 2024-09-21T10:02:23.585 INFO:tasks.rados.rados.0.smithi033.stdout:3442: snap_create 2024-09-21T10:02:23.589 INFO:tasks.rados.rados.0.smithi033.stdout:3439: finishing write tid 1 to smithi03385152-22 2024-09-21T10:02:23.589 INFO:tasks.rados.rados.0.smithi033.stdout:3439: finishing write tid 2 to smithi03385152-22 2024-09-21T10:02:23.589 INFO:tasks.rados.rados.0.smithi033.stdout:3439: finishing write tid 3 to smithi03385152-22 2024-09-21T10:02:23.591 INFO:tasks.rados.rados.0.smithi033.stdout:3438: finishing copy_from racing read to smithi03385152-6 2024-09-21T10:02:23.593 INFO:tasks.rados.rados.0.smithi033.stdout:3439: finishing write tid 4 to smithi03385152-22 2024-09-21T10:02:23.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2401 (ObjNum 1070 snap 313 seq_num 1070) dirty exists 2024-09-21T10:02:23.593 INFO:tasks.rados.rados.0.smithi033.stdout:3439: left oid 22 (ObjNum 1070 snap 313 seq_num 1070) 2024-09-21T10:02:23.593 INFO:tasks.rados.rados.0.smithi033.stdout:3438: finishing copy_from to smithi03385152-6 2024-09-21T10:02:23.593 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 2475 (ObjNum 1061 snap 311 seq_num 1061) dirty exists 2024-09-21T10:02:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:23 smithi033 ceph-mon[98546]: pgmap v694: 121 pgs: 121 active+clean; 197 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 10 op/s 2024-09-21T10:02:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:23 smithi033 ceph-mon[98546]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T10:02:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:23 smithi033 ceph-mon[98546]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T10:02:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:23 smithi033 ceph-mon[96568]: pgmap v694: 121 pgs: 121 active+clean; 197 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 10 op/s 2024-09-21T10:02:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:23 smithi033 ceph-mon[96568]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T10:02:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:23 smithi033 ceph-mon[96568]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T10:02:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:23 smithi105 ceph-mon[86649]: pgmap v694: 121 pgs: 121 active+clean; 197 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.9 MiB/s wr, 10 op/s 2024-09-21T10:02:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:23 smithi105 ceph-mon[86649]: osdmap e708: 8 total, 8 up, 8 in 2024-09-21T10:02:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:23 smithi105 ceph-mon[86649]: osdmap e709: 8 total, 8 up, 8 in 2024-09-21T10:02:24.565 INFO:tasks.rados.rados.0.smithi033.stdout:3438: done (2 left) 2024-09-21T10:02:24.565 INFO:tasks.rados.rados.0.smithi033.stdout:3439: done (1 left) 2024-09-21T10:02:24.565 INFO:tasks.rados.rados.0.smithi033.stdout:3442: done (0 left) 2024-09-21T10:02:24.565 INFO:tasks.rados.rados.0.smithi033.stdout:3443: read oid 43 snap -1 2024-09-21T10:02:24.565 INFO:tasks.rados.rados.0.smithi033.stdout:3443: expect (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:02:24.565 INFO:tasks.rados.rados.0.smithi033.stdout:3444: setattr oid 35 current snap is 314 2024-09-21T10:02:24.568 INFO:tasks.rados.rados.0.smithi033.stdout:3445: rmattr oid 6 current snap is 314 2024-09-21T10:02:24.568 INFO:tasks.rados.rados.0.smithi033.stdout:3445: done (2 left) 2024-09-21T10:02:24.568 INFO:tasks.rados.rados.0.smithi033.stdout:3446: read oid 26 snap 313 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3446: expect deleted 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3447: read oid 50 snap -1 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3447: expect (ObjNum 965 snap 283 seq_num 965) 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3448: read oid 31 snap -1 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3448: expect (ObjNum 1067 snap 313 seq_num 1067) 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3449: read oid 45 snap -1 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3449: expect (ObjNum 1045 snap 307 seq_num 1045) 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3450: rollback oid 1 current snap is 314 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 1 to 313 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 314 2024-09-21T10:02:24.569 INFO:tasks.rados.rados.0.smithi033.stdout:3451: seq_num 1072 ranges {3653632=761856,4415488=401408} 2024-09-21T10:02:24.594 INFO:tasks.rados.rados.0.smithi033.stdout:3451: writing smithi03385152-12 from 3653632 to 4415488 tid 1 2024-09-21T10:02:24.596 INFO:tasks.rados.rados.0.smithi033.stdout:3451: writing smithi03385152-12 from 4415488 to 4816896 tid 2 2024-09-21T10:02:24.596 INFO:tasks.rados.rados.0.smithi033.stdout:3452: delete oid 13 current snap is 314 2024-09-21T10:02:24.596 INFO:tasks.rados.rados.0.smithi033.stdout:3450: finishing rollback tid 0 to smithi03385152-1 2024-09-21T10:02:24.596 INFO:tasks.rados.rados.0.smithi033.stdout:3450: finishing rollback tid 1 to smithi03385152-1 2024-09-21T10:02:24.596 INFO:tasks.rados.rados.0.smithi033.stdout:3450: finishing rollback tid 2 to smithi03385152-1 2024-09-21T10:02:24.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1630 (ObjNum 986 snap 287 seq_num 986) dirty exists 2024-09-21T10:02:24.596 INFO:tasks.rados.rados.0.smithi033.stdout:3447: expect (ObjNum 965 snap 283 seq_num 965) 2024-09-21T10:02:24.775 INFO:tasks.rados.rados.0.smithi033.stdout:3443: expect (ObjNum 1020 snap 302 seq_num 1020) 2024-09-21T10:02:25.237 INFO:tasks.rados.rados.0.smithi033.stdout:3451: finishing write tid 1 to smithi03385152-12 2024-09-21T10:02:25.237 INFO:tasks.rados.rados.0.smithi033.stdout:3451: finishing write tid 2 to smithi03385152-12 2024-09-21T10:02:25.237 INFO:tasks.rados.rados.0.smithi033.stdout:3451: finishing write tid 3 to smithi03385152-12 2024-09-21T10:02:25.237 INFO:tasks.rados.rados.0.smithi033.stdout:3451: finishing write tid 4 to smithi03385152-12 2024-09-21T10:02:25.237 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2462 (ObjNum 1072 snap 314 seq_num 1072) dirty exists 2024-09-21T10:02:25.237 INFO:tasks.rados.rados.0.smithi033.stdout:3451: left oid 12 (ObjNum 1072 snap 314 seq_num 1072) 2024-09-21T10:02:25.237 INFO:tasks.rados.rados.0.smithi033.stdout:3449: expect (ObjNum 1045 snap 307 seq_num 1045) 2024-09-21T10:02:25.710 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2517 (ObjNum 1060 snap 311 seq_num 1060) dirty exists 2024-09-21T10:02:25.711 INFO:tasks.rados.rados.0.smithi033.stdout:3448: expect (ObjNum 1067 snap 313 seq_num 1067) 2024-09-21T10:02:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:25 smithi033 ceph-mon[98546]: pgmap v697: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.2 MiB/s wr, 14 op/s 2024-09-21T10:02:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:25 smithi033 ceph-mon[98546]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T10:02:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:25 smithi033 ceph-mon[96568]: pgmap v697: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.2 MiB/s wr, 14 op/s 2024-09-21T10:02:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:25 smithi033 ceph-mon[96568]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T10:02:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:25 smithi105 ceph-mon[86649]: pgmap v697: 121 pgs: 121 active+clean; 192 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 4.2 MiB/s wr, 14 op/s 2024-09-21T10:02:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:25 smithi105 ceph-mon[86649]: osdmap e710: 8 total, 8 up, 8 in 2024-09-21T10:02:27.245 INFO:tasks.rados.rados.0.smithi033.stdout:3443: done (8 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3444: done (7 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3446: done (6 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3447: done (5 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3448: done (4 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3449: done (3 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3450: done (2 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3451: done (1 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3452: done (0 left) 2024-09-21T10:02:27.246 INFO:tasks.rados.rados.0.smithi033.stdout:3453: delete oid 29 current snap is 314 2024-09-21T10:02:27.248 INFO:tasks.rados.rados.0.smithi033.stdout:3453: done (0 left) 2024-09-21T10:02:27.249 INFO:tasks.rados.rados.0.smithi033.stdout:3454: rollback oid 32 current snap is 314 2024-09-21T10:02:27.249 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 32 to 313 2024-09-21T10:02:27.249 INFO:tasks.rados.rados.0.smithi033.stdout:3455: read oid 10 snap -1 2024-09-21T10:02:27.249 INFO:tasks.rados.rados.0.smithi033.stdout:3455: expect deleted 2024-09-21T10:02:27.249 INFO:tasks.rados.rados.0.smithi033.stdout:3456: copy_from oid 48 from oid 24 current snap is 314 2024-09-21T10:02:27.249 INFO:tasks.rados.rados.0.smithi033.stdout:3457: snap_create 2024-09-21T10:02:27.252 INFO:tasks.rados.rados.0.smithi033.stdout:3454: finishing rollback tid 0 to smithi03385152-32 2024-09-21T10:02:27.252 INFO:tasks.rados.rados.0.smithi033.stdout:3454: finishing rollback tid 1 to smithi03385152-32 2024-09-21T10:02:27.253 INFO:tasks.rados.rados.0.smithi033.stdout:3454: finishing rollback tid 2 to smithi03385152-32 2024-09-21T10:02:27.253 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1597 (ObjNum 1065 snap 313 seq_num 1065) dirty exists 2024-09-21T10:02:27.258 INFO:tasks.rados.rados.0.smithi033.stdout:3456: finishing copy_from racing read to smithi03385152-48 2024-09-21T10:02:27.260 INFO:tasks.rados.rados.0.smithi033.stdout:3456: finishing copy_from to smithi03385152-48 2024-09-21T10:02:27.260 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 2264 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:02:27.569 INFO:tasks.rados.rados.0.smithi033.stdout:3454: done (3 left) 2024-09-21T10:02:27.570 INFO:tasks.rados.rados.0.smithi033.stdout:3455: done (2 left) 2024-09-21T10:02:27.570 INFO:tasks.rados.rados.0.smithi033.stdout:3456: done (1 left) 2024-09-21T10:02:27.570 INFO:tasks.rados.rados.0.smithi033.stdout:3457: done (0 left) 2024-09-21T10:02:27.570 INFO:tasks.rados.rados.0.smithi033.stdout:3458: snap_create 2024-09-21T10:02:27.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:27 smithi033 ceph-mon[98546]: pgmap v699: 121 pgs: 121 active+clean; 183 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T10:02:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:27 smithi033 ceph-mon[96568]: pgmap v699: 121 pgs: 121 active+clean; 183 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T10:02:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:27 smithi105 ceph-mon[86649]: pgmap v699: 121 pgs: 121 active+clean; 183 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 4.0 MiB/s wr, 19 op/s 2024-09-21T10:02:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:3458: done (0 left) 2024-09-21T10:02:28.560 INFO:tasks.rados.rados.0.smithi033.stdout:3459: snap_create 2024-09-21T10:02:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:28 smithi033 ceph-mon[98546]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T10:02:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:28 smithi033 ceph-mon[98546]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T10:02:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:28 smithi033 ceph-mon[96568]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T10:02:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:28 smithi033 ceph-mon[96568]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T10:02:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:28 smithi105 ceph-mon[86649]: osdmap e711: 8 total, 8 up, 8 in 2024-09-21T10:02:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:28 smithi105 ceph-mon[86649]: osdmap e712: 8 total, 8 up, 8 in 2024-09-21T10:02:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:3459: done (0 left) 2024-09-21T10:02:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:3460: rmattr oid 31 current snap is 317 2024-09-21T10:02:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 317 2024-09-21T10:02:29.562 INFO:tasks.rados.rados.0.smithi033.stdout:3461: seq_num 1075 ranges {933888=409600,1343488=671744,2015232=131072} 2024-09-21T10:02:29.569 INFO:tasks.rados.rados.0.smithi033.stdout:3461: writing smithi03385152-37 from 933888 to 1343488 tid 1 2024-09-21T10:02:29.573 INFO:tasks.rados.rados.0.smithi033.stdout:3461: writing smithi03385152-37 from 1343488 to 2015232 tid 2 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:3461: writing smithi03385152-37 from 2015232 to 2146304 tid 3 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:3462: copy_from oid 2 from oid 39 current snap is 317 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2523 (ObjNum 1067 snap 313 seq_num 1067) dirty exists 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:3460: done (2 left) 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:3463: rollback oid 14 current snap is 317 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 14 to 315 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:3464: read oid 28 snap -1 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:3464: expect (ObjNum 1048 snap 308 seq_num 1048) 2024-09-21T10:02:29.575 INFO:tasks.rados.rados.0.smithi033.stdout:3465: delete oid 26 current snap is 317 2024-09-21T10:02:29.577 INFO:tasks.rados.rados.0.smithi033.stdout:3461: finishing write tid 1 to smithi03385152-37 2024-09-21T10:02:29.579 INFO:tasks.rados.rados.0.smithi033.stdout:3461: finishing write tid 2 to smithi03385152-37 2024-09-21T10:02:29.580 INFO:tasks.rados.rados.0.smithi033.stdout:3461: finishing write tid 3 to smithi03385152-37 2024-09-21T10:02:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:3461: finishing write tid 4 to smithi03385152-37 2024-09-21T10:02:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:3463: finishing rollback tid 0 to smithi03385152-14 2024-09-21T10:02:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:3465: done (4 left) 2024-09-21T10:02:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:3466: read oid 33 snap 316 2024-09-21T10:02:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:3466: expect (ObjNum 1016 snap 299 seq_num 1016) 2024-09-21T10:02:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:3467: copy_from oid 5 from oid 22 current snap is 317 2024-09-21T10:02:29.582 INFO:tasks.rados.rados.0.smithi033.stdout:3463: finishing rollback tid 1 to smithi03385152-14 2024-09-21T10:02:29.582 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 317 2024-09-21T10:02:29.582 INFO:tasks.rados.rados.0.smithi033.stdout:3468: seq_num 1077 ranges {0=442368,442368=434176,876544=368640} 2024-09-21T10:02:29.583 INFO:tasks.rados.rados.0.smithi033.stdout:3468: writing smithi03385152-23 from 0 to 442368 tid 1 2024-09-21T10:02:29.587 INFO:tasks.rados.rados.0.smithi033.stdout:3468: writing smithi03385152-23 from 442368 to 876544 tid 2 2024-09-21T10:02:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3468: writing smithi03385152-23 from 876544 to 1245184 tid 3 2024-09-21T10:02:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 317 2024-09-21T10:02:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3463: finishing rollback tid 2 to smithi03385152-14 2024-09-21T10:02:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2404 (ObjNum 952 snap 279 seq_num 952) dirty exists 2024-09-21T10:02:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3461: finishing write tid 5 to smithi03385152-37 2024-09-21T10:02:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 2392 (ObjNum 1075 snap 317 seq_num 1075) dirty exists 2024-09-21T10:02:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3461: left oid 37 (ObjNum 1075 snap 317 seq_num 1075) 2024-09-21T10:02:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3468: finishing write tid 1 to smithi03385152-23 2024-09-21T10:02:29.590 INFO:tasks.rados.rados.0.smithi033.stdout:3469: seq_num 1078 ranges {4997120=458752,5455872=720896,6176768=81920} 2024-09-21T10:02:29.621 INFO:tasks.rados.rados.0.smithi033.stdout:3469: writing smithi03385152-30 from 4997120 to 5455872 tid 1 2024-09-21T10:02:29.626 INFO:tasks.rados.rados.0.smithi033.stdout:3469: writing smithi03385152-30 from 5455872 to 6176768 tid 2 2024-09-21T10:02:29.627 INFO:tasks.rados.rados.0.smithi033.stdout:3469: writing smithi03385152-30 from 6176768 to 6258688 tid 3 2024-09-21T10:02:29.627 INFO:tasks.rados.rados.0.smithi033.stdout:3461: done (7 left) 2024-09-21T10:02:29.627 INFO:tasks.rados.rados.0.smithi033.stdout:3463: done (6 left) 2024-09-21T10:02:29.627 INFO:tasks.rados.rados.0.smithi033.stdout:3470: copy_from oid 48 from oid 19 current snap is 317 2024-09-21T10:02:29.627 INFO:tasks.rados.rados.0.smithi033.stdout:3468: finishing write tid 2 to smithi03385152-23 2024-09-21T10:02:29.627 INFO:tasks.rados.rados.0.smithi033.stdout:3471: snap_create 2024-09-21T10:02:29.628 INFO:tasks.rados.rados.0.smithi033.stdout:3468: finishing write tid 3 to smithi03385152-23 2024-09-21T10:02:29.628 INFO:tasks.rados.rados.0.smithi033.stdout:3468: finishing write tid 4 to smithi03385152-23 2024-09-21T10:02:29.628 INFO:tasks.rados.rados.0.smithi033.stdout:3468: finishing write tid 5 to smithi03385152-23 2024-09-21T10:02:29.628 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 2437 (ObjNum 1077 snap 317 seq_num 1077) dirty exists 2024-09-21T10:02:29.628 INFO:tasks.rados.rados.0.smithi033.stdout:3468: left oid 23 (ObjNum 1077 snap 317 seq_num 1077) 2024-09-21T10:02:29.628 INFO:tasks.rados.rados.0.smithi033.stdout:3466: expect (ObjNum 1016 snap 299 seq_num 1016) 2024-09-21T10:02:29.728 INFO:tasks.rados.rados.0.smithi033.stdout:3464: expect (ObjNum 1048 snap 308 seq_num 1048) 2024-09-21T10:02:29.912 INFO:tasks.rados.rados.0.smithi033.stdout:3467: finishing copy_from racing read to smithi03385152-5 2024-09-21T10:02:29.912 INFO:tasks.rados.rados.0.smithi033.stdout:3467: finishing copy_from to smithi03385152-5 2024-09-21T10:02:29.912 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2401 (ObjNum 1070 snap 313 seq_num 1070) dirty exists 2024-09-21T10:02:29.912 INFO:tasks.rados.rados.0.smithi033.stdout:3462: finishing copy_from racing read to smithi03385152-2 2024-09-21T10:02:29.912 INFO:tasks.rados.rados.0.smithi033.stdout:3462: finishing copy_from to smithi03385152-2 2024-09-21T10:02:29.912 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 2391 (ObjNum 1052 snap 310 seq_num 1052) dirty exists 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:3469: finishing write tid 1 to smithi03385152-30 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:3469: finishing write tid 2 to smithi03385152-30 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:3469: finishing write tid 3 to smithi03385152-30 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:3469: finishing write tid 4 to smithi03385152-30 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:3469: finishing write tid 5 to smithi03385152-30 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2528 (ObjNum 1078 snap 317 seq_num 1078) dirty exists 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:3469: left oid 30 (ObjNum 1078 snap 317 seq_num 1078) 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:3470: finishing copy_from racing read to smithi03385152-48 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:3470: finishing copy_from to smithi03385152-48 2024-09-21T10:02:29.913 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 2269 (ObjNum 1016 snap 299 seq_num 1016) dirty exists 2024-09-21T10:02:29.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:29 smithi105 ceph-mon[86649]: pgmap v701: 121 pgs: 121 active+clean; 184 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.5 MiB/s wr, 17 op/s 2024-09-21T10:02:29.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:29 smithi105 ceph-mon[86649]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T10:02:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:29 smithi033 ceph-mon[98546]: pgmap v701: 121 pgs: 121 active+clean; 184 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.5 MiB/s wr, 17 op/s 2024-09-21T10:02:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:29 smithi033 ceph-mon[98546]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T10:02:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:29 smithi033 ceph-mon[96568]: pgmap v701: 121 pgs: 121 active+clean; 184 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 3.5 MiB/s wr, 17 op/s 2024-09-21T10:02:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:29 smithi033 ceph-mon[96568]: osdmap e713: 8 total, 8 up, 8 in 2024-09-21T10:02:30.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:02:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:02:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:30.583 INFO:tasks.rados.rados.0.smithi033.stdout:3462: done (7 left) 2024-09-21T10:02:30.583 INFO:tasks.rados.rados.0.smithi033.stdout:3464: done (6 left) 2024-09-21T10:02:30.583 INFO:tasks.rados.rados.0.smithi033.stdout:3466: done (5 left) 2024-09-21T10:02:30.583 INFO:tasks.rados.rados.0.smithi033.stdout:3467: done (4 left) 2024-09-21T10:02:30.583 INFO:tasks.rados.rados.0.smithi033.stdout:3468: done (3 left) 2024-09-21T10:02:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3469: done (2 left) 2024-09-21T10:02:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3470: done (1 left) 2024-09-21T10:02:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3471: done (0 left) 2024-09-21T10:02:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3472: read oid 38 snap 316 2024-09-21T10:02:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3472: expect (ObjNum 1012 snap 299 seq_num 1012) 2024-09-21T10:02:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3473: delete oid 20 current snap is 318 2024-09-21T10:02:30.586 INFO:tasks.rados.rados.0.smithi033.stdout:3473: done (1 left) 2024-09-21T10:02:30.587 INFO:tasks.rados.rados.0.smithi033.stdout:3474: snap_create 2024-09-21T10:02:30.588 INFO:tasks.rados.rados.0.smithi033.stdout:3472: expect (ObjNum 1012 snap 299 seq_num 1012) 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:3472: done (1 left) 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:3474: done (0 left) 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:3475: rollback oid 31 current snap is 319 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 31 to 318 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:3476: read oid 45 snap -1 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:3476: expect (ObjNum 1045 snap 307 seq_num 1045) 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:3477: read oid 1 snap -1 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:3477: expect (ObjNum 986 snap 287 seq_num 986) 2024-09-21T10:02:31.585 INFO:tasks.rados.rados.0.smithi033.stdout:3478: snap_remove snap 315 2024-09-21T10:02:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:02:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:02:31] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-09-21T10:02:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:31 smithi033 ceph-mon[98546]: pgmap v704: 121 pgs: 121 active+clean; 185 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T10:02:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:31 smithi033 ceph-mon[98546]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T10:02:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:31 smithi033 ceph-mon[96568]: pgmap v704: 121 pgs: 121 active+clean; 185 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T10:02:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:31 smithi033 ceph-mon[96568]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T10:02:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:31 smithi105 ceph-mon[86649]: pgmap v704: 121 pgs: 121 active+clean; 185 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 21 MiB/s rd, 1.2 MiB/s wr, 12 op/s 2024-09-21T10:02:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:31 smithi105 ceph-mon[86649]: osdmap e714: 8 total, 8 up, 8 in 2024-09-21T10:02:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:32.588 INFO:tasks.rados.rados.0.smithi033.stdout:3478: done (3 left) 2024-09-21T10:02:32.588 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 41 current snap is 319 2024-09-21T10:02:32.588 INFO:tasks.rados.rados.0.smithi033.stdout:3479: seq_num 1080 ranges {2883584=491520,3375104=540672,3915776=262144} 2024-09-21T10:02:32.606 INFO:tasks.rados.rados.0.smithi033.stdout:3479: writing smithi03385152-41 from 2883584 to 3375104 tid 1 2024-09-21T10:02:32.610 INFO:tasks.rados.rados.0.smithi033.stdout:3479: writing smithi03385152-41 from 3375104 to 3915776 tid 2 2024-09-21T10:02:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:3479: writing smithi03385152-41 from 3915776 to 4177920 tid 3 2024-09-21T10:02:32.611 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 319 2024-09-21T10:02:32.612 INFO:tasks.rados.rados.0.smithi033.stdout:3480: seq_num 1081 ranges {3203072=507904,3710976=425984,4136960=376832} 2024-09-21T10:02:32.632 INFO:tasks.rados.rados.0.smithi033.stdout:3480: writing smithi03385152-22 from 3203072 to 3710976 tid 1 2024-09-21T10:02:32.634 INFO:tasks.rados.rados.0.smithi033.stdout:3480: writing smithi03385152-22 from 3710976 to 4136960 tid 2 2024-09-21T10:02:32.637 INFO:tasks.rados.rados.0.smithi033.stdout:3480: writing smithi03385152-22 from 4136960 to 4513792 tid 3 2024-09-21T10:02:32.637 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 19 current snap is 319 2024-09-21T10:02:32.637 INFO:tasks.rados.rados.0.smithi033.stdout:3475: finishing rollback tid 0 to smithi03385152-31 2024-09-21T10:02:32.637 INFO:tasks.rados.rados.0.smithi033.stdout:3475: finishing rollback tid 1 to smithi03385152-31 2024-09-21T10:02:32.637 INFO:tasks.rados.rados.0.smithi033.stdout:3475: finishing rollback tid 2 to smithi03385152-31 2024-09-21T10:02:32.637 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2532 (ObjNum 1067 snap 313 seq_num 1067) dirty exists 2024-09-21T10:02:32.637 INFO:tasks.rados.rados.0.smithi033.stdout:3477: expect (ObjNum 986 snap 287 seq_num 986) 2024-09-21T10:02:32.697 INFO:tasks.rados.rados.0.smithi033.stdout:3476: expect (ObjNum 1045 snap 307 seq_num 1045) 2024-09-21T10:02:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:32 smithi033 ceph-mon[98546]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T10:02:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:32 smithi033 ceph-mon[96568]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T10:02:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:32 smithi105 ceph-mon[86649]: osdmap e715: 8 total, 8 up, 8 in 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:3479: finishing write tid 1 to smithi03385152-41 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:3479: finishing write tid 2 to smithi03385152-41 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:3479: finishing write tid 3 to smithi03385152-41 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:3479: finishing write tid 4 to smithi03385152-41 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:3479: finishing write tid 5 to smithi03385152-41 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 2397 (ObjNum 1080 snap 319 seq_num 1080) dirty exists 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:3479: left oid 41 (ObjNum 1080 snap 319 seq_num 1080) 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:3480: finishing write tid 1 to smithi03385152-22 2024-09-21T10:02:33.171 INFO:tasks.rados.rados.0.smithi033.stdout:3481: seq_num 1082 ranges {2416640=524288,2940928=704512,3645440=98304} 2024-09-21T10:02:33.186 INFO:tasks.rados.rados.0.smithi033.stdout:3481: writing smithi03385152-19 from 2416640 to 2940928 tid 1 2024-09-21T10:02:33.190 INFO:tasks.rados.rados.0.smithi033.stdout:3481: writing smithi03385152-19 from 2940928 to 3645440 tid 2 2024-09-21T10:02:33.191 INFO:tasks.rados.rados.0.smithi033.stdout:3481: writing smithi03385152-19 from 3645440 to 3743744 tid 3 2024-09-21T10:02:33.191 INFO:tasks.rados.rados.0.smithi033.stdout:3475: done (5 left) 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3476: done (4 left) 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3477: done (3 left) 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3479: done (2 left) 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 319 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3480: finishing write tid 2 to smithi03385152-22 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3480: finishing write tid 3 to smithi03385152-22 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3480: finishing write tid 4 to smithi03385152-22 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3480: finishing write tid 5 to smithi03385152-22 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2408 (ObjNum 1081 snap 319 seq_num 1081) dirty exists 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3480: left oid 22 (ObjNum 1081 snap 319 seq_num 1081) 2024-09-21T10:02:33.192 INFO:tasks.rados.rados.0.smithi033.stdout:3482: seq_num 1083 ranges {1245184=540672,1785856=589824,2375680=212992} 2024-09-21T10:02:33.201 INFO:tasks.rados.rados.0.smithi033.stdout:3482: writing smithi03385152-23 from 1245184 to 1785856 tid 1 2024-09-21T10:02:33.204 INFO:tasks.rados.rados.0.smithi033.stdout:3482: writing smithi03385152-23 from 1785856 to 2375680 tid 2 2024-09-21T10:02:33.206 INFO:tasks.rados.rados.0.smithi033.stdout:3482: writing smithi03385152-23 from 2375680 to 2588672 tid 3 2024-09-21T10:02:33.206 INFO:tasks.rados.rados.0.smithi033.stdout:3480: done (2 left) 2024-09-21T10:02:33.206 INFO:tasks.rados.rados.0.smithi033.stdout:3483: read oid 16 snap -1 2024-09-21T10:02:33.206 INFO:tasks.rados.rados.0.smithi033.stdout:3483: expect (ObjNum 1050 snap 308 seq_num 1050) 2024-09-21T10:02:33.206 INFO:tasks.rados.rados.0.smithi033.stdout:3484: snap_remove snap 313 2024-09-21T10:02:33.564 INFO:tasks.rados.rados.0.smithi033.stdout:3484: done (3 left) 2024-09-21T10:02:33.564 INFO:tasks.rados.rados.0.smithi033.stdout:3485: delete oid 13 current snap is 319 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3481: finishing write tid 1 to smithi03385152-19 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3481: finishing write tid 2 to smithi03385152-19 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3481: finishing write tid 3 to smithi03385152-19 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3481: finishing write tid 4 to smithi03385152-19 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3481: finishing write tid 5 to smithi03385152-19 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2276 (ObjNum 1082 snap 319 seq_num 1082) dirty exists 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3481: left oid 19 (ObjNum 1082 snap 319 seq_num 1082) 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3482: finishing write tid 1 to smithi03385152-23 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3482: finishing write tid 2 to smithi03385152-23 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3482: finishing write tid 3 to smithi03385152-23 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3482: finishing write tid 4 to smithi03385152-23 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3482: finishing write tid 5 to smithi03385152-23 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 2442 (ObjNum 1083 snap 319 seq_num 1083) dirty exists 2024-09-21T10:02:33.565 INFO:tasks.rados.rados.0.smithi033.stdout:3482: left oid 23 (ObjNum 1083 snap 319 seq_num 1083) 2024-09-21T10:02:33.566 INFO:tasks.rados.rados.0.smithi033.stdout:3483: expect (ObjNum 1050 snap 308 seq_num 1050) 2024-09-21T10:02:33.608 INFO:tasks.rados.rados.0.smithi033.stdout:3481: done (3 left) 2024-09-21T10:02:33.608 INFO:tasks.rados.rados.0.smithi033.stdout:3482: done (2 left) 2024-09-21T10:02:33.608 INFO:tasks.rados.rados.0.smithi033.stdout:3483: done (1 left) 2024-09-21T10:02:33.608 INFO:tasks.rados.rados.0.smithi033.stdout:3485: done (0 left) 2024-09-21T10:02:33.608 INFO:tasks.rados.rados.0.smithi033.stdout:3486: rollback oid 21 current snap is 319 2024-09-21T10:02:33.609 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 314 2024-09-21T10:02:33.609 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 319 2024-09-21T10:02:33.609 INFO:tasks.rados.rados.0.smithi033.stdout:3487: seq_num 1085 ranges {7405568=581632,7987200=761856,8749056=40960} 2024-09-21T10:02:33.652 INFO:tasks.rados.rados.0.smithi033.stdout:3487: writing smithi03385152-49 from 7405568 to 7987200 tid 1 2024-09-21T10:02:33.656 INFO:tasks.rados.rados.0.smithi033.stdout:3487: writing smithi03385152-49 from 7987200 to 8749056 tid 2 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:3487: writing smithi03385152-49 from 8749056 to 8790016 tid 3 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:3486: finishing rollback tid 0 to smithi03385152-21 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:3486: finishing rollback tid 1 to smithi03385152-21 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:3488: read oid 4 snap 317 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:3488: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:3489: delete oid 5 current snap is 319 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:3486: finishing rollback tid 2 to smithi03385152-21 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 2445 (ObjNum 1050 snap 308 seq_num 1050) dirty exists 2024-09-21T10:02:33.657 INFO:tasks.rados.rados.0.smithi033.stdout:3487: finishing write tid 1 to smithi03385152-49 2024-09-21T10:02:33.662 INFO:tasks.rados.rados.0.smithi033.stdout:3486: done (3 left) 2024-09-21T10:02:33.662 INFO:tasks.rados.rados.0.smithi033.stdout:3489: done (2 left) 2024-09-21T10:02:33.662 INFO:tasks.rados.rados.0.smithi033.stdout:3490: snap_create 2024-09-21T10:02:33.662 INFO:tasks.rados.rados.0.smithi033.stdout:3487: finishing write tid 2 to smithi03385152-49 2024-09-21T10:02:33.662 INFO:tasks.rados.rados.0.smithi033.stdout:3487: finishing write tid 3 to smithi03385152-49 2024-09-21T10:02:33.663 INFO:tasks.rados.rados.0.smithi033.stdout:3487: finishing write tid 4 to smithi03385152-49 2024-09-21T10:02:33.665 INFO:tasks.rados.rados.0.smithi033.stdout:3487: finishing write tid 5 to smithi03385152-49 2024-09-21T10:02:33.665 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 2466 (ObjNum 1085 snap 319 seq_num 1085) dirty exists 2024-09-21T10:02:33.665 INFO:tasks.rados.rados.0.smithi033.stdout:3487: left oid 49 (ObjNum 1085 snap 319 seq_num 1085) 2024-09-21T10:02:33.671 INFO:tasks.rados.rados.0.smithi033.stdout:3488: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:33 smithi033 ceph-mon[98546]: pgmap v707: 121 pgs: 121 active+clean; 199 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 3.8 MiB/s wr, 8 op/s 2024-09-21T10:02:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:33 smithi033 ceph-mon[98546]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T10:02:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:33 smithi033 ceph-mon[98546]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T10:02:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:33 smithi033 ceph-mon[96568]: pgmap v707: 121 pgs: 121 active+clean; 199 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 3.8 MiB/s wr, 8 op/s 2024-09-21T10:02:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:33 smithi033 ceph-mon[96568]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T10:02:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:33 smithi033 ceph-mon[96568]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T10:02:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:33 smithi105 ceph-mon[86649]: pgmap v707: 121 pgs: 121 active+clean; 199 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.9 MiB/s rd, 3.8 MiB/s wr, 8 op/s 2024-09-21T10:02:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:33 smithi105 ceph-mon[86649]: osdmap e716: 8 total, 8 up, 8 in 2024-09-21T10:02:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:33 smithi105 ceph-mon[86649]: osdmap e717: 8 total, 8 up, 8 in 2024-09-21T10:02:34.612 INFO:tasks.rados.rados.0.smithi033.stdout:3487: done (2 left) 2024-09-21T10:02:34.612 INFO:tasks.rados.rados.0.smithi033.stdout:3488: done (1 left) 2024-09-21T10:02:34.612 INFO:tasks.rados.rados.0.smithi033.stdout:3490: done (0 left) 2024-09-21T10:02:34.612 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 320 2024-09-21T10:02:34.612 INFO:tasks.rados.rados.0.smithi033.stdout:3491: seq_num 1087 ranges {7012352=614400,7626752=532480,8159232=270336} 2024-09-21T10:02:34.653 INFO:tasks.rados.rados.0.smithi033.stdout:3491: writing smithi03385152-45 from 7012352 to 7626752 tid 1 2024-09-21T10:02:34.656 INFO:tasks.rados.rados.0.smithi033.stdout:3491: writing smithi03385152-45 from 7626752 to 8159232 tid 2 2024-09-21T10:02:34.657 INFO:tasks.rados.rados.0.smithi033.stdout:3491: writing smithi03385152-45 from 8159232 to 8429568 tid 3 2024-09-21T10:02:34.658 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 320 2024-09-21T10:02:34.658 INFO:tasks.rados.rados.0.smithi033.stdout:3492: seq_num 1088 ranges {2236416=630784,2867200=466944,3334144=335872} 2024-09-21T10:02:34.673 INFO:tasks.rados.rados.0.smithi033.stdout:3492: writing smithi03385152-1 from 2236416 to 2867200 tid 1 2024-09-21T10:02:34.676 INFO:tasks.rados.rados.0.smithi033.stdout:3492: writing smithi03385152-1 from 2867200 to 3334144 tid 2 2024-09-21T10:02:34.678 INFO:tasks.rados.rados.0.smithi033.stdout:3492: writing smithi03385152-1 from 3334144 to 3670016 tid 3 2024-09-21T10:02:34.678 INFO:tasks.rados.rados.0.smithi033.stdout:3493: snap_remove snap 319 2024-09-21T10:02:35.613 INFO:tasks.rados.rados.0.smithi033.stdout:3491: finishing write tid 1 to smithi03385152-45 2024-09-21T10:02:35.613 INFO:tasks.rados.rados.0.smithi033.stdout:3491: finishing write tid 2 to smithi03385152-45 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3491: finishing write tid 3 to smithi03385152-45 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3491: finishing write tid 4 to smithi03385152-45 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3491: finishing write tid 5 to smithi03385152-45 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2412 (ObjNum 1087 snap 320 seq_num 1087) dirty exists 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3491: left oid 45 (ObjNum 1087 snap 320 seq_num 1087) 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3492: finishing write tid 1 to smithi03385152-1 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3492: finishing write tid 2 to smithi03385152-1 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3492: finishing write tid 3 to smithi03385152-1 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3492: finishing write tid 4 to smithi03385152-1 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:3492: finishing write tid 5 to smithi03385152-1 2024-09-21T10:02:35.614 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1634 (ObjNum 1088 snap 320 seq_num 1088) dirty exists 2024-09-21T10:02:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3492: left oid 1 (ObjNum 1088 snap 320 seq_num 1088) 2024-09-21T10:02:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3491: done (2 left) 2024-09-21T10:02:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3492: done (1 left) 2024-09-21T10:02:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3493: done (0 left) 2024-09-21T10:02:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3494: read oid 32 snap 316 2024-09-21T10:02:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3494: expect (ObjNum 1065 snap 313 seq_num 1065) 2024-09-21T10:02:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3495: snap_create 2024-09-21T10:02:35.631 INFO:tasks.rados.rados.0.smithi033.stdout:3494: expect (ObjNum 1065 snap 313 seq_num 1065) 2024-09-21T10:02:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:35 smithi033 ceph-mon[98546]: pgmap v710: 121 pgs: 121 active+clean; 206 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 5.2 MiB/s wr, 12 op/s 2024-09-21T10:02:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:35 smithi033 ceph-mon[98546]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T10:02:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:35 smithi033 ceph-mon[96568]: pgmap v710: 121 pgs: 121 active+clean; 206 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 5.2 MiB/s wr, 12 op/s 2024-09-21T10:02:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:35 smithi033 ceph-mon[96568]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T10:02:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:35 smithi105 ceph-mon[86649]: pgmap v710: 121 pgs: 121 active+clean; 206 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 5.2 MiB/s wr, 12 op/s 2024-09-21T10:02:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:35 smithi105 ceph-mon[86649]: osdmap e718: 8 total, 8 up, 8 in 2024-09-21T10:02:36.620 INFO:tasks.rados.rados.0.smithi033.stdout:3494: done (1 left) 2024-09-21T10:02:36.620 INFO:tasks.rados.rados.0.smithi033.stdout:3495: done (0 left) 2024-09-21T10:02:36.620 INFO:tasks.rados.rados.0.smithi033.stdout:3496: snap_remove snap 317 2024-09-21T10:02:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:36 smithi033 ceph-mon[98546]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T10:02:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:36 smithi033 ceph-mon[96568]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T10:02:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:36 smithi105 ceph-mon[86649]: osdmap e719: 8 total, 8 up, 8 in 2024-09-21T10:02:37.615 INFO:tasks.rados.rados.0.smithi033.stdout:3496: done (0 left) 2024-09-21T10:02:37.615 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 4 current snap is 321 2024-09-21T10:02:37.615 INFO:tasks.rados.rados.0.smithi033.stdout:3497: seq_num 1089 ranges {2023424=647168,2670592=745472,3416064=57344} 2024-09-21T10:02:37.631 INFO:tasks.rados.rados.0.smithi033.stdout:3497: writing smithi03385152-4 from 2023424 to 2670592 tid 1 2024-09-21T10:02:37.635 INFO:tasks.rados.rados.0.smithi033.stdout:3497: writing smithi03385152-4 from 2670592 to 3416064 tid 2 2024-09-21T10:02:37.636 INFO:tasks.rados.rados.0.smithi033.stdout:3497: writing smithi03385152-4 from 3416064 to 3473408 tid 3 2024-09-21T10:02:37.636 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 321 2024-09-21T10:02:37.636 INFO:tasks.rados.rados.0.smithi033.stdout:3498: seq_num 1090 ranges {3653632=663552,4317184=630784,4947968=172032} 2024-09-21T10:02:37.659 INFO:tasks.rados.rados.0.smithi033.stdout:3498: writing smithi03385152-28 from 3653632 to 4317184 tid 1 2024-09-21T10:02:37.663 INFO:tasks.rados.rados.0.smithi033.stdout:3498: writing smithi03385152-28 from 4317184 to 4947968 tid 2 2024-09-21T10:02:37.664 INFO:tasks.rados.rados.0.smithi033.stdout:3498: writing smithi03385152-28 from 4947968 to 5120000 tid 3 2024-09-21T10:02:37.664 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 321 2024-09-21T10:02:37.664 INFO:tasks.rados.rados.0.smithi033.stdout:3497: finishing write tid 1 to smithi03385152-4 2024-09-21T10:02:37.664 INFO:tasks.rados.rados.0.smithi033.stdout:3499: seq_num 1091 ranges {3440640=679936,4120576=516096,4636672=286720} 2024-09-21T10:02:37.686 INFO:tasks.rados.rados.0.smithi033.stdout:3499: writing smithi03385152-32 from 3440640 to 4120576 tid 1 2024-09-21T10:02:37.689 INFO:tasks.rados.rados.0.smithi033.stdout:3499: writing smithi03385152-32 from 4120576 to 4636672 tid 2 2024-09-21T10:02:37.691 INFO:tasks.rados.rados.0.smithi033.stdout:3499: writing smithi03385152-32 from 4636672 to 4923392 tid 3 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3500: delete oid 36 current snap is 321 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3497: finishing write tid 2 to smithi03385152-4 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3497: finishing write tid 3 to smithi03385152-4 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3497: finishing write tid 4 to smithi03385152-4 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3497: finishing write tid 5 to smithi03385152-4 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 2377 (ObjNum 1089 snap 321 seq_num 1089) dirty exists 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3497: left oid 4 (ObjNum 1089 snap 321 seq_num 1089) 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3498: finishing write tid 1 to smithi03385152-28 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3498: finishing write tid 2 to smithi03385152-28 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3498: finishing write tid 3 to smithi03385152-28 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3498: finishing write tid 4 to smithi03385152-28 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3498: finishing write tid 5 to smithi03385152-28 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1834 (ObjNum 1090 snap 321 seq_num 1090) dirty exists 2024-09-21T10:02:37.692 INFO:tasks.rados.rados.0.smithi033.stdout:3498: left oid 28 (ObjNum 1090 snap 321 seq_num 1090) 2024-09-21T10:02:37.693 INFO:tasks.rados.rados.0.smithi033.stdout:3499: finishing write tid 1 to smithi03385152-32 2024-09-21T10:02:37.693 INFO:tasks.rados.rados.0.smithi033.stdout:3499: finishing write tid 2 to smithi03385152-32 2024-09-21T10:02:37.693 INFO:tasks.rados.rados.0.smithi033.stdout:3499: finishing write tid 3 to smithi03385152-32 2024-09-21T10:02:37.694 INFO:tasks.rados.rados.0.smithi033.stdout:3499: finishing write tid 4 to smithi03385152-32 2024-09-21T10:02:37.694 INFO:tasks.rados.rados.0.smithi033.stdout:3497: done (3 left) 2024-09-21T10:02:37.694 INFO:tasks.rados.rados.0.smithi033.stdout:3498: done (2 left) 2024-09-21T10:02:37.694 INFO:tasks.rados.rados.0.smithi033.stdout:3500: done (1 left) 2024-09-21T10:02:37.695 INFO:tasks.rados.rados.0.smithi033.stdout:3501: snap_create 2024-09-21T10:02:37.695 INFO:tasks.rados.rados.0.smithi033.stdout:3499: finishing write tid 5 to smithi03385152-32 2024-09-21T10:02:37.695 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1601 (ObjNum 1091 snap 321 seq_num 1091) dirty exists 2024-09-21T10:02:37.695 INFO:tasks.rados.rados.0.smithi033.stdout:3499: left oid 32 (ObjNum 1091 snap 321 seq_num 1091) 2024-09-21T10:02:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:37 smithi033 ceph-mon[98546]: pgmap v713: 121 pgs: 121 active+clean; 189 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-21T10:02:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:37 smithi033 ceph-mon[98546]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T10:02:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:37 smithi033 ceph-mon[98546]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T10:02:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:37 smithi033 ceph-mon[96568]: pgmap v713: 121 pgs: 121 active+clean; 189 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-21T10:02:37.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:37 smithi033 ceph-mon[96568]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T10:02:37.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:37 smithi033 ceph-mon[96568]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T10:02:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:37 smithi105 ceph-mon[86649]: pgmap v713: 121 pgs: 121 active+clean; 189 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 3.0 MiB/s wr, 14 op/s 2024-09-21T10:02:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:37 smithi105 ceph-mon[86649]: osdmap e720: 8 total, 8 up, 8 in 2024-09-21T10:02:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:37 smithi105 ceph-mon[86649]: osdmap e721: 8 total, 8 up, 8 in 2024-09-21T10:02:38.642 INFO:tasks.rados.rados.0.smithi033.stdout:3499: done (1 left) 2024-09-21T10:02:38.642 INFO:tasks.rados.rados.0.smithi033.stdout:3501: done (0 left) 2024-09-21T10:02:38.642 INFO:tasks.rados.rados.0.smithi033.stdout:3502: read oid 21 snap 314 2024-09-21T10:02:38.642 INFO:tasks.rados.rados.0.smithi033.stdout:3502: expect (ObjNum 1050 snap 308 seq_num 1050) 2024-09-21T10:02:38.642 INFO:tasks.rados.rados.0.smithi033.stdout:3503: delete oid 3 current snap is 322 2024-09-21T10:02:38.647 INFO:tasks.rados.rados.0.smithi033.stdout:3502: expect (ObjNum 1050 snap 308 seq_num 1050) 2024-09-21T10:02:38.688 INFO:tasks.rados.rados.0.smithi033.stdout:3502: done (1 left) 2024-09-21T10:02:38.688 INFO:tasks.rados.rados.0.smithi033.stdout:3503: done (0 left) 2024-09-21T10:02:38.688 INFO:tasks.rados.rados.0.smithi033.stdout:3504: read oid 23 snap 314 2024-09-21T10:02:38.688 INFO:tasks.rados.rados.0.smithi033.stdout:3504: expect deleted 2024-09-21T10:02:38.688 INFO:tasks.rados.rados.0.smithi033.stdout:3505: setattr oid 29 current snap is 322 2024-09-21T10:02:38.689 INFO:tasks.rados.rados.0.smithi033.stdout:3504: done (1 left) 2024-09-21T10:02:38.689 INFO:tasks.rados.rados.0.smithi033.stdout:3506: copy_from oid 26 from oid 10 current snap is 322 2024-09-21T10:02:38.689 INFO:tasks.rados.rados.0.smithi033.stdout:3507: delete oid 9 current snap is 322 2024-09-21T10:02:38.692 INFO:tasks.rados.rados.0.smithi033.stdout:3506: finishing copy_from to smithi03385152-26 2024-09-21T10:02:38.693 INFO:tasks.rados.rados.0.smithi033.stdout:3506: got expected ENOENT (src dne) 2024-09-21T10:02:38.693 INFO:tasks.rados.rados.0.smithi033.stdout:3506: finishing copy_from racing read to smithi03385152-26 2024-09-21T10:02:38.693 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2551 (ObjNum 65 snap 0 seq_num 72668112) dirty exists 2024-09-21T10:02:38.694 INFO:tasks.rados.rados.0.smithi033.stdout:3505: done (2 left) 2024-09-21T10:02:38.694 INFO:tasks.rados.rados.0.smithi033.stdout:3506: done (1 left) 2024-09-21T10:02:38.694 INFO:tasks.rados.rados.0.smithi033.stdout:3507: done (0 left) 2024-09-21T10:02:38.694 INFO:tasks.rados.rados.0.smithi033.stdout:3508: snap_create 2024-09-21T10:02:39.648 INFO:tasks.rados.rados.0.smithi033.stdout:3508: done (0 left) 2024-09-21T10:02:39.649 INFO:tasks.rados.rados.0.smithi033.stdout:3509: copy_from oid 14 from oid 19 current snap is 323 2024-09-21T10:02:39.649 INFO:tasks.rados.rados.0.smithi033.stdout:3510: rmattr oid 49 current snap is 323 2024-09-21T10:02:39.649 INFO:tasks.rados.rados.0.smithi033.stdout:3511: delete oid 31 current snap is 323 2024-09-21T10:02:39.655 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 2467 (ObjNum 1085 snap 319 seq_num 1085) dirty exists 2024-09-21T10:02:39.656 INFO:tasks.rados.rados.0.smithi033.stdout:3510: done (2 left) 2024-09-21T10:02:39.656 INFO:tasks.rados.rados.0.smithi033.stdout:3511: done (1 left) 2024-09-21T10:02:39.656 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 323 2024-09-21T10:02:39.656 INFO:tasks.rados.rados.0.smithi033.stdout:3512: seq_num 1096 ranges {8232960=761856,8994816=786432,9781248=16384} 2024-09-21T10:02:39.705 INFO:tasks.rados.rados.0.smithi033.stdout:3512: writing smithi03385152-39 from 8232960 to 8994816 tid 1 2024-09-21T10:02:39.709 INFO:tasks.rados.rados.0.smithi033.stdout:3512: writing smithi03385152-39 from 8994816 to 9781248 tid 2 2024-09-21T10:02:39.710 INFO:tasks.rados.rados.0.smithi033.stdout:3512: writing smithi03385152-39 from 9781248 to 9797632 tid 3 2024-09-21T10:02:39.710 INFO:tasks.rados.rados.0.smithi033.stdout:3513: delete oid 5 current snap is 323 2024-09-21T10:02:39.710 INFO:tasks.rados.rados.0.smithi033.stdout:3509: finishing copy_from racing read to smithi03385152-14 2024-09-21T10:02:39.710 INFO:tasks.rados.rados.0.smithi033.stdout:3509: finishing copy_from to smithi03385152-14 2024-09-21T10:02:39.710 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2276 (ObjNum 1082 snap 319 seq_num 1082) dirty exists 2024-09-21T10:02:39.710 INFO:tasks.rados.rados.0.smithi033.stdout:3512: finishing write tid 1 to smithi03385152-39 2024-09-21T10:02:39.713 INFO:tasks.rados.rados.0.smithi033.stdout:3512: finishing write tid 2 to smithi03385152-39 2024-09-21T10:02:39.713 INFO:tasks.rados.rados.0.smithi033.stdout:3512: finishing write tid 3 to smithi03385152-39 2024-09-21T10:02:39.714 INFO:tasks.rados.rados.0.smithi033.stdout:3512: finishing write tid 4 to smithi03385152-39 2024-09-21T10:02:39.715 INFO:tasks.rados.rados.0.smithi033.stdout:3509: done (2 left) 2024-09-21T10:02:39.715 INFO:tasks.rados.rados.0.smithi033.stdout:3513: done (1 left) 2024-09-21T10:02:39.715 INFO:tasks.rados.rados.0.smithi033.stdout:3514: rmattr oid 47 current snap is 323 2024-09-21T10:02:39.715 INFO:tasks.rados.rados.0.smithi033.stdout:3514: done (1 left) 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3515: read oid 47 snap 318 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3515: expect (ObjNum 1011 snap 298 seq_num 1011) 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3516: read oid 15 snap -1 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3516: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3517: rollback oid 37 current snap is 323 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 320 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3518: copy_from oid 49 from oid 14 current snap is 323 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3519: read oid 40 snap 320 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3519: expect (ObjNum 1030 snap 303 seq_num 1030) 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3520: read oid 12 snap -1 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3520: expect (ObjNum 1072 snap 314 seq_num 1072) 2024-09-21T10:02:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3521: snap_remove snap 316 2024-09-21T10:02:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:39 smithi033 ceph-mon[98546]: pgmap v716: 121 pgs: 121 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T10:02:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:39 smithi033 ceph-mon[98546]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T10:02:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:39 smithi033 ceph-mon[96568]: pgmap v716: 121 pgs: 121 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T10:02:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:39 smithi033 ceph-mon[96568]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T10:02:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:39 smithi105 ceph-mon[86649]: pgmap v716: 121 pgs: 121 active+clean; 190 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 1.6 MiB/s wr, 10 op/s 2024-09-21T10:02:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:39 smithi105 ceph-mon[86649]: osdmap e722: 8 total, 8 up, 8 in 2024-09-21T10:02:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:02:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:02:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:40.656 INFO:tasks.rados.rados.0.smithi033.stdout:3521: done (7 left) 2024-09-21T10:02:40.656 INFO:tasks.rados.rados.0.smithi033.stdout:3522: copy_from oid 50 from oid 24 current snap is 323 2024-09-21T10:02:40.656 INFO:tasks.rados.rados.0.smithi033.stdout:3523: rollback oid 45 current snap is 323 2024-09-21T10:02:40.656 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 45 to 318 2024-09-21T10:02:40.656 INFO:tasks.rados.rados.0.smithi033.stdout:3524: rollback oid 18 current snap is 323 2024-09-21T10:02:40.656 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 18 to 322 2024-09-21T10:02:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3525: rmattr oid 1 current snap is 323 2024-09-21T10:02:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3525: done (10 left) 2024-09-21T10:02:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3526: read oid 19 snap -1 2024-09-21T10:02:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3526: expect (ObjNum 1082 snap 319 seq_num 1082) 2024-09-21T10:02:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 323 2024-09-21T10:02:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3527: seq_num 1098 ranges {1269760=794624,2064384=557056,2621440=245760} 2024-09-21T10:02:40.669 INFO:tasks.rados.rados.0.smithi033.stdout:3527: writing smithi03385152-21 from 1269760 to 2064384 tid 1 2024-09-21T10:02:40.671 INFO:tasks.rados.rados.0.smithi033.stdout:3527: writing smithi03385152-21 from 2064384 to 2621440 tid 2 2024-09-21T10:02:40.673 INFO:tasks.rados.rados.0.smithi033.stdout:3527: writing smithi03385152-21 from 2621440 to 2867200 tid 3 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:3528: copy_from oid 10 from oid 32 current snap is 323 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:3512: finishing write tid 5 to smithi03385152-39 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 2406 (ObjNum 1096 snap 323 seq_num 1096) dirty exists 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:3512: left oid 39 (ObjNum 1096 snap 323 seq_num 1096) 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:3517: finishing rollback tid 0 to smithi03385152-37 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:3517: finishing rollback tid 1 to smithi03385152-37 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:3517: finishing rollback tid 2 to smithi03385152-37 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 2395 (ObjNum 1075 snap 317 seq_num 1075) dirty exists 2024-09-21T10:02:40.674 INFO:tasks.rados.rados.0.smithi033.stdout:3516: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:40.734 INFO:tasks.rados.rados.0.smithi033.stdout:3518: finishing copy_from racing read to smithi03385152-49 2024-09-21T10:02:40.735 INFO:tasks.rados.rados.0.smithi033.stdout:3519: expect (ObjNum 1030 snap 303 seq_num 1030) 2024-09-21T10:02:40.942 INFO:tasks.rados.rados.0.smithi033.stdout:3515: expect (ObjNum 1011 snap 298 seq_num 1011) 2024-09-21T10:02:40.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:40 smithi033 ceph-mon[96568]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T10:02:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:40 smithi033 ceph-mon[98546]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T10:02:40.996 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:02:40 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[113392]: 2024-09-21T10:02:40.683+0000 7fe79c71a640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 192 after trimming was started 2024-09-21T10:02:40.996 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:02:40 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[113392]: 2024-09-21T10:02:40.688+0000 7fe79a716640 -1 snap_mapper.get_next_objects_to_trim::New Clone-Objects were added to Snap 192 after trimming was started 2024-09-21T10:02:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:40 smithi105 ceph-mon[86649]: osdmap e723: 8 total, 8 up, 8 in 2024-09-21T10:02:41.179 INFO:tasks.rados.rados.0.smithi033.stdout:3520: expect (ObjNum 1072 snap 314 seq_num 1072) 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3518: finishing copy_from to smithi03385152-49 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 2276 (ObjNum 1082 snap 319 seq_num 1082) dirty exists 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3524: finishing rollback tid 0 to smithi03385152-18 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3524: finishing rollback tid 1 to smithi03385152-18 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3524: finishing rollback tid 2 to smithi03385152-18 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1828 (ObjNum 1068 snap 313 seq_num 1068) dirty exists 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3523: finishing rollback tid 0 to smithi03385152-45 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3523: finishing rollback tid 1 to smithi03385152-45 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3512: done (12 left) 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3515: done (11 left) 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3516: done (10 left) 2024-09-21T10:02:41.445 INFO:tasks.rados.rados.0.smithi033.stdout:3517: done (9 left) 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3518: done (8 left) 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3519: done (7 left) 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3520: done (6 left) 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3524: done (5 left) 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 323 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3523: finishing rollback tid 2 to smithi03385152-45 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2415 (ObjNum 1045 snap 307 seq_num 1045) dirty exists 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3527: finishing write tid 1 to smithi03385152-21 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3522: finishing copy_from racing read to smithi03385152-50 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3527: finishing write tid 2 to smithi03385152-21 2024-09-21T10:02:41.446 INFO:tasks.rados.rados.0.smithi033.stdout:3529: seq_num 1099 ranges {0=409600,409600=442368,851968=720896,1572864=40960} 2024-09-21T10:02:41.447 INFO:tasks.rados.rados.0.smithi033.stdout:3529: writing smithi03385152-9 from 0 to 409600 tid 1 2024-09-21T10:02:41.448 INFO:tasks.rados.rados.0.smithi033.stdout:3529: writing smithi03385152-9 from 409600 to 851968 tid 2 2024-09-21T10:02:41.454 INFO:tasks.rados.rados.0.smithi033.stdout:3529: writing smithi03385152-9 from 851968 to 1572864 tid 3 2024-09-21T10:02:41.455 INFO:tasks.rados.rados.0.smithi033.stdout:3529: writing smithi03385152-9 from 1572864 to 1613824 tid 4 2024-09-21T10:02:41.455 INFO:tasks.rados.rados.0.smithi033.stdout:3523: done (5 left) 2024-09-21T10:02:41.456 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 323 2024-09-21T10:02:41.456 INFO:tasks.rados.rados.0.smithi033.stdout:3530: seq_num 1100 ranges {3743744=425984,4169728=729088,4898816=475136} 2024-09-21T10:02:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:3530: writing smithi03385152-49 from 3743744 to 4169728 tid 1 2024-09-21T10:02:41.480 INFO:tasks.rados.rados.0.smithi033.stdout:3530: writing smithi03385152-49 from 4169728 to 4898816 tid 2 2024-09-21T10:02:41.483 INFO:tasks.rados.rados.0.smithi033.stdout:3530: writing smithi03385152-49 from 4898816 to 5373952 tid 3 2024-09-21T10:02:41.483 INFO:tasks.rados.rados.0.smithi033.stdout:3531: rollback oid 37 current snap is 323 2024-09-21T10:02:41.483 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 37 to 314 2024-09-21T10:02:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3527: finishing write tid 3 to smithi03385152-21 2024-09-21T10:02:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3527: finishing write tid 4 to smithi03385152-21 2024-09-21T10:02:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3522: finishing copy_from to smithi03385152-50 2024-09-21T10:02:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 2264 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:02:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3527: finishing write tid 5 to smithi03385152-21 2024-09-21T10:02:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 2449 (ObjNum 1098 snap 323 seq_num 1098) dirty exists 2024-09-21T10:02:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3527: left oid 21 (ObjNum 1098 snap 323 seq_num 1098) 2024-09-21T10:02:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3526: expect (ObjNum 1082 snap 319 seq_num 1082) 2024-09-21T10:02:41.658 INFO:tasks.rados.rados.0.smithi033.stdout:3528: finishing copy_from racing read to smithi03385152-10 2024-09-21T10:02:41.658 INFO:tasks.rados.rados.0.smithi033.stdout:3528: finishing copy_from to smithi03385152-10 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1601 (ObjNum 1091 snap 321 seq_num 1091) dirty exists 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3522: done (6 left) 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3526: done (5 left) 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3527: done (4 left) 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3528: done (3 left) 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3532: rmattr oid 2 current snap is 323 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3532: done (3 left) 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3533: setattr oid 12 current snap is 323 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3529: finishing write tid 1 to smithi03385152-9 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3529: finishing write tid 2 to smithi03385152-9 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3529: finishing write tid 3 to smithi03385152-9 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3529: finishing write tid 4 to smithi03385152-9 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3529: finishing write tid 5 to smithi03385152-9 2024-09-21T10:02:41.659 INFO:tasks.rados.rados.0.smithi033.stdout:3529: finishing write tid 6 to smithi03385152-9 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 2403 (ObjNum 1099 snap 323 seq_num 1099) dirty exists 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3529: left oid 9 (ObjNum 1099 snap 323 seq_num 1099) 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3530: finishing write tid 1 to smithi03385152-49 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3530: finishing write tid 2 to smithi03385152-49 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3531: finishing rollback tid 0 to smithi03385152-37 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3531: finishing rollback tid 1 to smithi03385152-37 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3531: finishing rollback tid 2 to smithi03385152-37 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 2398 (ObjNum 915 snap 265 seq_num 915) dirty exists 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3530: finishing write tid 3 to smithi03385152-49 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3530: finishing write tid 4 to smithi03385152-49 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3530: finishing write tid 5 to smithi03385152-49 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 2471 (ObjNum 1100 snap 323 seq_num 1100) dirty exists 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3530: left oid 49 (ObjNum 1100 snap 323 seq_num 1100) 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3529: done (3 left) 2024-09-21T10:02:41.660 INFO:tasks.rados.rados.0.smithi033.stdout:3530: done (2 left) 2024-09-21T10:02:41.661 INFO:tasks.rados.rados.0.smithi033.stdout:3531: done (1 left) 2024-09-21T10:02:41.661 INFO:tasks.rados.rados.0.smithi033.stdout:3534: snap_create 2024-09-21T10:02:41.661 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2472 (ObjNum 1072 snap 314 seq_num 1072) dirty exists 2024-09-21T10:02:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:41 smithi033 ceph-mon[96568]: pgmap v719: 121 pgs: 121 active+clean; 193 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-21T10:02:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:41 smithi033 ceph-mon[96568]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T10:02:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:41 smithi033 ceph-mon[98546]: pgmap v719: 121 pgs: 121 active+clean; 193 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-21T10:02:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:41 smithi033 ceph-mon[98546]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T10:02:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:02:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:02:41] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-09-21T10:02:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:41 smithi105 ceph-mon[86649]: pgmap v719: 121 pgs: 121 active+clean; 193 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.0 MiB/s wr, 6 op/s 2024-09-21T10:02:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:41 smithi105 ceph-mon[86649]: osdmap e724: 8 total, 8 up, 8 in 2024-09-21T10:02:42.661 INFO:tasks.rados.rados.0.smithi033.stdout:3533: done (1 left) 2024-09-21T10:02:42.661 INFO:tasks.rados.rados.0.smithi033.stdout:3534: done (0 left) 2024-09-21T10:02:42.662 INFO:tasks.rados.rados.0.smithi033.stdout:3535: delete oid 42 current snap is 324 2024-09-21T10:02:42.667 INFO:tasks.rados.rados.0.smithi033.stdout:3535: done (0 left) 2024-09-21T10:02:42.667 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 26 current snap is 324 2024-09-21T10:02:42.667 INFO:tasks.rados.rados.0.smithi033.stdout:3536: seq_num 1102 ranges {0=458752,458752=491520,950272=491520,1441792=221184} 2024-09-21T10:02:42.669 INFO:tasks.rados.rados.0.smithi033.stdout:3536: writing smithi03385152-26 from 0 to 458752 tid 1 2024-09-21T10:02:42.672 INFO:tasks.rados.rados.0.smithi033.stdout:3536: writing smithi03385152-26 from 458752 to 950272 tid 2 2024-09-21T10:02:42.675 INFO:tasks.rados.rados.0.smithi033.stdout:3536: writing smithi03385152-26 from 950272 to 1441792 tid 3 2024-09-21T10:02:42.676 INFO:tasks.rados.rados.0.smithi033.stdout:3536: writing smithi03385152-26 from 1441792 to 1662976 tid 4 2024-09-21T10:02:42.676 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 324 2024-09-21T10:02:42.676 INFO:tasks.rados.rados.0.smithi033.stdout:3537: seq_num 1103 ranges {0=475136,475136=425984,901120=696320,1597440=81920} 2024-09-21T10:02:42.679 INFO:tasks.rados.rados.0.smithi033.stdout:3537: writing smithi03385152-42 from 0 to 475136 tid 1 2024-09-21T10:02:42.681 INFO:tasks.rados.rados.0.smithi033.stdout:3537: writing smithi03385152-42 from 475136 to 901120 tid 2 2024-09-21T10:02:42.686 INFO:tasks.rados.rados.0.smithi033.stdout:3537: writing smithi03385152-42 from 901120 to 1597440 tid 3 2024-09-21T10:02:42.687 INFO:tasks.rados.rados.0.smithi033.stdout:3537: writing smithi03385152-42 from 1597440 to 1679360 tid 4 2024-09-21T10:02:42.687 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 324 2024-09-21T10:02:42.687 INFO:tasks.rados.rados.0.smithi033.stdout:3536: finishing write tid 1 to smithi03385152-26 2024-09-21T10:02:42.687 INFO:tasks.rados.rados.0.smithi033.stdout:3538: seq_num 1104 ranges {4513792=499712,5013504=712704,5726208=491520} 2024-09-21T10:02:42.714 INFO:tasks.rados.rados.0.smithi033.stdout:3538: writing smithi03385152-22 from 4513792 to 5013504 tid 1 2024-09-21T10:02:42.718 INFO:tasks.rados.rados.0.smithi033.stdout:3538: writing smithi03385152-22 from 5013504 to 5726208 tid 2 2024-09-21T10:02:42.722 INFO:tasks.rados.rados.0.smithi033.stdout:3538: writing smithi03385152-22 from 5726208 to 6217728 tid 3 2024-09-21T10:02:42.722 INFO:tasks.rados.rados.0.smithi033.stdout:3539: delete oid 44 current snap is 324 2024-09-21T10:02:42.722 INFO:tasks.rados.rados.0.smithi033.stdout:3536: finishing write tid 2 to smithi03385152-26 2024-09-21T10:02:42.722 INFO:tasks.rados.rados.0.smithi033.stdout:3536: finishing write tid 3 to smithi03385152-26 2024-09-21T10:02:42.722 INFO:tasks.rados.rados.0.smithi033.stdout:3536: finishing write tid 4 to smithi03385152-26 2024-09-21T10:02:42.722 INFO:tasks.rados.rados.0.smithi033.stdout:3536: finishing write tid 5 to smithi03385152-26 2024-09-21T10:02:42.722 INFO:tasks.rados.rados.0.smithi033.stdout:3537: finishing write tid 1 to smithi03385152-42 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3536: finishing write tid 6 to smithi03385152-26 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 26 v 2477 (ObjNum 1102 snap 324 seq_num 1102) dirty exists 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3536: left oid 26 (ObjNum 1102 snap 324 seq_num 1102) 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3537: finishing write tid 2 to smithi03385152-42 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3537: finishing write tid 3 to smithi03385152-42 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3537: finishing write tid 4 to smithi03385152-42 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3537: finishing write tid 5 to smithi03385152-42 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3537: finishing write tid 6 to smithi03385152-42 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 2409 (ObjNum 1103 snap 324 seq_num 1103) dirty exists 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3537: left oid 42 (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:02:42.723 INFO:tasks.rados.rados.0.smithi033.stdout:3538: finishing write tid 1 to smithi03385152-22 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:3536: done (3 left) 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:3537: done (2 left) 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:3539: done (1 left) 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:3540: read oid 18 snap -1 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:3540: expect (ObjNum 1068 snap 313 seq_num 1068) 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:3541: read oid 34 snap -1 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:3541: expect (ObjNum 1035 snap 305 seq_num 1035) 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 14 current snap is 324 2024-09-21T10:02:42.724 INFO:tasks.rados.rados.0.smithi033.stdout:3542: seq_num 1106 ranges {3743744=532480,4276224=483328,4759552=475136,5234688=245760} 2024-09-21T10:02:42.747 INFO:tasks.rados.rados.0.smithi033.stdout:3542: writing smithi03385152-14 from 3743744 to 4276224 tid 1 2024-09-21T10:02:42.750 INFO:tasks.rados.rados.0.smithi033.stdout:3542: writing smithi03385152-14 from 4276224 to 4759552 tid 2 2024-09-21T10:02:42.752 INFO:tasks.rados.rados.0.smithi033.stdout:3542: writing smithi03385152-14 from 4759552 to 5234688 tid 3 2024-09-21T10:02:42.754 INFO:tasks.rados.rados.0.smithi033.stdout:3542: writing smithi03385152-14 from 5234688 to 5480448 tid 4 2024-09-21T10:02:42.754 INFO:tasks.rados.rados.0.smithi033.stdout:3543: copy_from oid 31 from oid 38 current snap is 324 2024-09-21T10:02:42.754 INFO:tasks.rados.rados.0.smithi033.stdout:3538: finishing write tid 2 to smithi03385152-22 2024-09-21T10:02:42.754 INFO:tasks.rados.rados.0.smithi033.stdout:3538: finishing write tid 3 to smithi03385152-22 2024-09-21T10:02:42.754 INFO:tasks.rados.rados.0.smithi033.stdout:3538: finishing write tid 4 to smithi03385152-22 2024-09-21T10:02:42.755 INFO:tasks.rados.rados.0.smithi033.stdout:3538: finishing write tid 5 to smithi03385152-22 2024-09-21T10:02:42.755 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2419 (ObjNum 1104 snap 324 seq_num 1104) dirty exists 2024-09-21T10:02:42.755 INFO:tasks.rados.rados.0.smithi033.stdout:3538: left oid 22 (ObjNum 1104 snap 324 seq_num 1104) 2024-09-21T10:02:42.755 INFO:tasks.rados.rados.0.smithi033.stdout:3540: expect (ObjNum 1068 snap 313 seq_num 1068) 2024-09-21T10:02:42.815 INFO:tasks.rados.rados.0.smithi033.stdout:3541: expect (ObjNum 1035 snap 305 seq_num 1035) 2024-09-21T10:02:42.836 INFO:tasks.rados.rados.0.smithi033.stdout:3542: finishing write tid 1 to smithi03385152-14 2024-09-21T10:02:42.836 INFO:tasks.rados.rados.0.smithi033.stdout:3542: finishing write tid 2 to smithi03385152-14 2024-09-21T10:02:42.836 INFO:tasks.rados.rados.0.smithi033.stdout:3542: finishing write tid 3 to smithi03385152-14 2024-09-21T10:02:42.836 INFO:tasks.rados.rados.0.smithi033.stdout:3542: finishing write tid 4 to smithi03385152-14 2024-09-21T10:02:42.836 INFO:tasks.rados.rados.0.smithi033.stdout:3542: finishing write tid 5 to smithi03385152-14 2024-09-21T10:02:42.836 INFO:tasks.rados.rados.0.smithi033.stdout:3542: finishing write tid 6 to smithi03385152-14 2024-09-21T10:02:42.836 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2424 (ObjNum 1106 snap 324 seq_num 1106) dirty exists 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3542: left oid 14 (ObjNum 1106 snap 324 seq_num 1106) 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3543: finishing copy_from racing read to smithi03385152-31 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3543: finishing copy_from to smithi03385152-31 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2378 (ObjNum 1012 snap 299 seq_num 1012) dirty exists 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3538: done (4 left) 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3540: done (3 left) 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3541: done (2 left) 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3542: done (1 left) 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3543: done (0 left) 2024-09-21T10:02:42.837 INFO:tasks.rados.rados.0.smithi033.stdout:3544: setattr oid 41 current snap is 324 2024-09-21T10:02:42.838 INFO:tasks.rados.rados.0.smithi033.stdout:3545: read oid 5 snap 322 2024-09-21T10:02:42.838 INFO:tasks.rados.rados.0.smithi033.stdout:3545: expect deleted 2024-09-21T10:02:42.838 INFO:tasks.rados.rados.0.smithi033.stdout:3546: snap_remove snap 321 2024-09-21T10:02:43.565 INFO:tasks.rados.rados.0.smithi033.stdout:3546: done (2 left) 2024-09-21T10:02:43.565 INFO:tasks.rados.rados.0.smithi033.stdout:3547: copy_from oid 14 from oid 37 current snap is 324 2024-09-21T10:02:43.565 INFO:tasks.rados.rados.0.smithi033.stdout:3548: snap_remove snap 323 2024-09-21T10:02:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:43 smithi033 ceph-mon[98546]: pgmap v721: 121 pgs: 121 active+clean; 198 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T10:02:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:43 smithi033 ceph-mon[98546]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T10:02:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:43 smithi033 ceph-mon[98546]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T10:02:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:43 smithi033 ceph-mon[96568]: pgmap v721: 121 pgs: 121 active+clean; 198 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T10:02:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:43 smithi033 ceph-mon[96568]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T10:02:43.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:43 smithi033 ceph-mon[96568]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T10:02:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:43 smithi105 ceph-mon[86649]: pgmap v721: 121 pgs: 121 active+clean; 198 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-09-21T10:02:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:43 smithi105 ceph-mon[86649]: osdmap e725: 8 total, 8 up, 8 in 2024-09-21T10:02:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:43 smithi105 ceph-mon[86649]: osdmap e726: 8 total, 8 up, 8 in 2024-09-21T10:02:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:3548: done (3 left) 2024-09-21T10:02:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:3549: read oid 2 snap -1 2024-09-21T10:02:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:3549: expect (ObjNum 1052 snap 310 seq_num 1052) 2024-09-21T10:02:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:3550: delete oid 49 current snap is 324 2024-09-21T10:02:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 2410 (ObjNum 1080 snap 319 seq_num 1080) dirty exists 2024-09-21T10:02:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:3547: finishing copy_from racing read to smithi03385152-14 2024-09-21T10:02:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:3547: finishing copy_from to smithi03385152-14 2024-09-21T10:02:44.567 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2398 (ObjNum 915 snap 265 seq_num 915) dirty exists 2024-09-21T10:02:44.575 INFO:tasks.rados.rados.0.smithi033.stdout:3544: done (4 left) 2024-09-21T10:02:44.576 INFO:tasks.rados.rados.0.smithi033.stdout:3545: done (3 left) 2024-09-21T10:02:44.576 INFO:tasks.rados.rados.0.smithi033.stdout:3547: done (2 left) 2024-09-21T10:02:44.576 INFO:tasks.rados.rados.0.smithi033.stdout:3550: done (1 left) 2024-09-21T10:02:44.576 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 324 2024-09-21T10:02:44.576 INFO:tasks.rados.rados.0.smithi033.stdout:3551: seq_num 1108 ranges {0=565248,565248=647168,1212416=483328,1695744=73728} 2024-09-21T10:02:44.577 INFO:tasks.rados.rados.0.smithi033.stdout:3551: writing smithi03385152-29 from 0 to 565248 tid 1 2024-09-21T10:02:44.581 INFO:tasks.rados.rados.0.smithi033.stdout:3551: writing smithi03385152-29 from 565248 to 1212416 tid 2 2024-09-21T10:02:44.584 INFO:tasks.rados.rados.0.smithi033.stdout:3551: writing smithi03385152-29 from 1212416 to 1695744 tid 3 2024-09-21T10:02:44.585 INFO:tasks.rados.rados.0.smithi033.stdout:3551: writing smithi03385152-29 from 1695744 to 1769472 tid 4 2024-09-21T10:02:44.585 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 324 2024-09-21T10:02:44.585 INFO:tasks.rados.rados.0.smithi033.stdout:3551: finishing write tid 1 to smithi03385152-29 2024-09-21T10:02:44.585 INFO:tasks.rados.rados.0.smithi033.stdout:3552: seq_num 1109 ranges {11280384=581632,11862016=532480,12394496=647168,13041664=24576} 2024-09-21T10:02:44.650 INFO:tasks.rados.rados.0.smithi033.stdout:3552: writing smithi03385152-35 from 11280384 to 11862016 tid 1 2024-09-21T10:02:44.652 INFO:tasks.rados.rados.0.smithi033.stdout:3552: writing smithi03385152-35 from 11862016 to 12394496 tid 2 2024-09-21T10:02:44.657 INFO:tasks.rados.rados.0.smithi033.stdout:3552: writing smithi03385152-35 from 12394496 to 13041664 tid 3 2024-09-21T10:02:44.657 INFO:tasks.rados.rados.0.smithi033.stdout:3552: writing smithi03385152-35 from 13041664 to 13066240 tid 4 2024-09-21T10:02:44.657 INFO:tasks.rados.rados.0.smithi033.stdout:3553: copy_from oid 17 from oid 42 current snap is 324 2024-09-21T10:02:44.657 INFO:tasks.rados.rados.0.smithi033.stdout:3551: finishing write tid 2 to smithi03385152-29 2024-09-21T10:02:44.657 INFO:tasks.rados.rados.0.smithi033.stdout:3551: finishing write tid 3 to smithi03385152-29 2024-09-21T10:02:44.657 INFO:tasks.rados.rados.0.smithi033.stdout:3551: finishing write tid 4 to smithi03385152-29 2024-09-21T10:02:44.657 INFO:tasks.rados.rados.0.smithi033.stdout:3551: finishing write tid 5 to smithi03385152-29 2024-09-21T10:02:44.658 INFO:tasks.rados.rados.0.smithi033.stdout:3551: finishing write tid 6 to smithi03385152-29 2024-09-21T10:02:44.658 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2566 (ObjNum 1108 snap 324 seq_num 1108) dirty exists 2024-09-21T10:02:44.658 INFO:tasks.rados.rados.0.smithi033.stdout:3551: left oid 29 (ObjNum 1108 snap 324 seq_num 1108) 2024-09-21T10:02:44.658 INFO:tasks.rados.rados.0.smithi033.stdout:3549: expect (ObjNum 1052 snap 310 seq_num 1052) 2024-09-21T10:02:45.265 INFO:tasks.rados.rados.0.smithi033.stdout:3552: finishing write tid 1 to smithi03385152-35 2024-09-21T10:02:45.265 INFO:tasks.rados.rados.0.smithi033.stdout:3552: finishing write tid 2 to smithi03385152-35 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3552: finishing write tid 3 to smithi03385152-35 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3552: finishing write tid 4 to smithi03385152-35 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3552: finishing write tid 5 to smithi03385152-35 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3552: finishing write tid 6 to smithi03385152-35 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2572 (ObjNum 1109 snap 324 seq_num 1109) dirty exists 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3552: left oid 35 (ObjNum 1109 snap 324 seq_num 1109) 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3553: finishing copy_from racing read to smithi03385152-17 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3553: finishing copy_from to smithi03385152-17 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2409 (ObjNum 1103 snap 324 seq_num 1103) dirty exists 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3549: done (3 left) 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3551: done (2 left) 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3552: done (1 left) 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3553: done (0 left) 2024-09-21T10:02:45.266 INFO:tasks.rados.rados.0.smithi033.stdout:3554: snap_remove snap 320 2024-09-21T10:02:45.666 INFO:tasks.rados.rados.0.smithi033.stdout:3554: done (0 left) 2024-09-21T10:02:45.666 INFO:tasks.rados.rados.0.smithi033.stdout:3555: rmattr oid 43 current snap is 324 2024-09-21T10:02:45.667 INFO:tasks.rados.rados.0.smithi033.stdout:3555: done (0 left) 2024-09-21T10:02:45.667 INFO:tasks.rados.rados.0.smithi033.stdout:3556: setattr oid 39 current snap is 324 2024-09-21T10:02:45.668 INFO:tasks.rados.rados.0.smithi033.stdout:3557: read oid 32 snap -1 2024-09-21T10:02:45.668 INFO:tasks.rados.rados.0.smithi033.stdout:3557: expect (ObjNum 1091 snap 321 seq_num 1091) 2024-09-21T10:02:45.668 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 324 2024-09-21T10:02:45.668 INFO:tasks.rados.rados.0.smithi033.stdout:3558: seq_num 1110 ranges {7012352=598016,7610368=466944,8077312=450560,8527872=286720} 2024-09-21T10:02:45.710 INFO:tasks.rados.rados.0.smithi033.stdout:3558: writing smithi03385152-45 from 7012352 to 7610368 tid 1 2024-09-21T10:02:45.713 INFO:tasks.rados.rados.0.smithi033.stdout:3558: writing smithi03385152-45 from 7610368 to 8077312 tid 2 2024-09-21T10:02:45.715 INFO:tasks.rados.rados.0.smithi033.stdout:3558: writing smithi03385152-45 from 8077312 to 8527872 tid 3 2024-09-21T10:02:45.717 INFO:tasks.rados.rados.0.smithi033.stdout:3558: writing smithi03385152-45 from 8527872 to 8814592 tid 4 2024-09-21T10:02:45.717 INFO:tasks.rados.rados.0.smithi033.stdout:3559: snap_remove snap 318 2024-09-21T10:02:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:45 smithi033 ceph-mon[98546]: pgmap v724: 121 pgs: 121 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.4 MiB/s wr, 28 op/s 2024-09-21T10:02:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:45 smithi033 ceph-mon[98546]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T10:02:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:45 smithi033 ceph-mon[96568]: pgmap v724: 121 pgs: 121 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.4 MiB/s wr, 28 op/s 2024-09-21T10:02:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:45 smithi033 ceph-mon[96568]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T10:02:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:45 smithi105 ceph-mon[86649]: pgmap v724: 121 pgs: 121 active+clean; 214 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.4 MiB/s wr, 28 op/s 2024-09-21T10:02:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:45 smithi105 ceph-mon[86649]: osdmap e727: 8 total, 8 up, 8 in 2024-09-21T10:02:46.676 INFO:tasks.rados.rados.0.smithi033.stdout:3559: done (3 left) 2024-09-21T10:02:46.676 INFO:tasks.rados.rados.0.smithi033.stdout:3560: delete oid 24 current snap is 324 2024-09-21T10:02:46.677 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 2407 (ObjNum 1096 snap 323 seq_num 1096) dirty exists 2024-09-21T10:02:46.677 INFO:tasks.rados.rados.0.smithi033.stdout:3557: expect (ObjNum 1091 snap 321 seq_num 1091) 2024-09-21T10:02:46.937 INFO:tasks.rados.rados.0.smithi033.stdout:3558: finishing write tid 1 to smithi03385152-45 2024-09-21T10:02:46.937 INFO:tasks.rados.rados.0.smithi033.stdout:3558: finishing write tid 2 to smithi03385152-45 2024-09-21T10:02:46.937 INFO:tasks.rados.rados.0.smithi033.stdout:3558: finishing write tid 3 to smithi03385152-45 2024-09-21T10:02:46.937 INFO:tasks.rados.rados.0.smithi033.stdout:3558: finishing write tid 4 to smithi03385152-45 2024-09-21T10:02:46.937 INFO:tasks.rados.rados.0.smithi033.stdout:3556: done (3 left) 2024-09-21T10:02:46.937 INFO:tasks.rados.rados.0.smithi033.stdout:3557: done (2 left) 2024-09-21T10:02:46.938 INFO:tasks.rados.rados.0.smithi033.stdout:3560: done (1 left) 2024-09-21T10:02:46.938 INFO:tasks.rados.rados.0.smithi033.stdout:3561: copy_from oid 1 from oid 10 current snap is 324 2024-09-21T10:02:46.938 INFO:tasks.rados.rados.0.smithi033.stdout:3558: finishing write tid 5 to smithi03385152-45 2024-09-21T10:02:46.938 INFO:tasks.rados.rados.0.smithi033.stdout:3558: finishing write tid 6 to smithi03385152-45 2024-09-21T10:02:46.938 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2429 (ObjNum 1110 snap 324 seq_num 1110) dirty exists 2024-09-21T10:02:46.938 INFO:tasks.rados.rados.0.smithi033.stdout:3558: left oid 45 (ObjNum 1110 snap 324 seq_num 1110) 2024-09-21T10:02:46.938 INFO:tasks.rados.rados.0.smithi033.stdout:3558: done (1 left) 2024-09-21T10:02:46.938 INFO:tasks.rados.rados.0.smithi033.stdout:3562: delete oid 41 current snap is 324 2024-09-21T10:02:46.940 INFO:tasks.rados.rados.0.smithi033.stdout:3562: done (1 left) 2024-09-21T10:02:46.940 INFO:tasks.rados.rados.0.smithi033.stdout:3563: copy_from oid 36 from oid 33 current snap is 324 2024-09-21T10:02:46.940 INFO:tasks.rados.rados.0.smithi033.stdout:3564: copy_from oid 37 from oid 6 current snap is 324 2024-09-21T10:02:46.940 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 39 current snap is 324 2024-09-21T10:02:46.940 INFO:tasks.rados.rados.0.smithi033.stdout:3565: seq_num 1113 ranges {9797632=647168,10444800=524288,10969088=622592,11591680=57344} 2024-09-21T10:02:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:46 smithi033 ceph-mon[98546]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T10:02:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:46 smithi033 ceph-mon[96568]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T10:02:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:46.998 INFO:tasks.rados.rados.0.smithi033.stdout:3565: writing smithi03385152-39 from 9797632 to 10444800 tid 1 2024-09-21T10:02:47.001 INFO:tasks.rados.rados.0.smithi033.stdout:3565: writing smithi03385152-39 from 10444800 to 10969088 tid 2 2024-09-21T10:02:47.005 INFO:tasks.rados.rados.0.smithi033.stdout:3565: writing smithi03385152-39 from 10969088 to 11591680 tid 3 2024-09-21T10:02:47.006 INFO:tasks.rados.rados.0.smithi033.stdout:3565: writing smithi03385152-39 from 11591680 to 11649024 tid 4 2024-09-21T10:02:47.006 INFO:tasks.rados.rados.0.smithi033.stdout:3566: rollback oid 4 current snap is 324 2024-09-21T10:02:47.006 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 4 to 322 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3563: finishing copy_from racing read to smithi03385152-36 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3563: finishing copy_from to smithi03385152-36 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2269 (ObjNum 1016 snap 299 seq_num 1016) dirty exists 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3561: finishing copy_from racing read to smithi03385152-1 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3564: finishing copy_from racing read to smithi03385152-37 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3564: finishing copy_from to smithi03385152-37 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 2475 (ObjNum 1061 snap 311 seq_num 1061) dirty exists 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3561: finishing copy_from to smithi03385152-1 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1601 (ObjNum 1091 snap 321 seq_num 1091) dirty exists 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3565: finishing write tid 1 to smithi03385152-39 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3565: finishing write tid 2 to smithi03385152-39 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3561: done (4 left) 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3563: done (3 left) 2024-09-21T10:02:47.007 INFO:tasks.rados.rados.0.smithi033.stdout:3564: done (2 left) 2024-09-21T10:02:47.008 INFO:tasks.rados.rados.0.smithi033.stdout:3567: delete oid 43 current snap is 324 2024-09-21T10:02:47.009 INFO:tasks.rados.rados.0.smithi033.stdout:3567: done (2 left) 2024-09-21T10:02:47.009 INFO:tasks.rados.rados.0.smithi033.stdout:3568: copy_from oid 26 from oid 5 current snap is 324 2024-09-21T10:02:47.009 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 35 current snap is 324 2024-09-21T10:02:47.009 INFO:tasks.rados.rados.0.smithi033.stdout:3569: seq_num 1115 ranges {13066240=679936,13746176=688128,14434304=516096} 2024-09-21T10:02:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:46 smithi105 ceph-mon[86649]: osdmap e728: 8 total, 8 up, 8 in 2024-09-21T10:02:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:02:47.084 INFO:tasks.rados.rados.0.smithi033.stdout:3569: writing smithi03385152-35 from 13066240 to 13746176 tid 1 2024-09-21T10:02:47.088 INFO:tasks.rados.rados.0.smithi033.stdout:3569: writing smithi03385152-35 from 13746176 to 14434304 tid 2 2024-09-21T10:02:47.092 INFO:tasks.rados.rados.0.smithi033.stdout:3569: writing smithi03385152-35 from 14434304 to 14950400 tid 3 2024-09-21T10:02:47.092 INFO:tasks.rados.rados.0.smithi033.stdout:3570: rollback oid 47 current snap is 324 2024-09-21T10:02:47.092 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 322 2024-09-21T10:02:47.092 INFO:tasks.rados.rados.0.smithi033.stdout:3565: finishing write tid 3 to smithi03385152-39 2024-09-21T10:02:47.092 INFO:tasks.rados.rados.0.smithi033.stdout:3566: finishing rollback tid 0 to smithi03385152-4 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3565: finishing write tid 4 to smithi03385152-39 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3565: finishing write tid 5 to smithi03385152-39 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3566: finishing rollback tid 1 to smithi03385152-4 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3566: finishing rollback tid 2 to smithi03385152-4 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 2381 (ObjNum 1089 snap 321 seq_num 1089) dirty exists 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3568: finishing copy_from to smithi03385152-26 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3568: got expected ENOENT (src dne) 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3568: finishing copy_from racing read to smithi03385152-26 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3565: finishing write tid 6 to smithi03385152-39 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 2412 (ObjNum 1113 snap 324 seq_num 1113) dirty exists 2024-09-21T10:02:47.093 INFO:tasks.rados.rados.0.smithi033.stdout:3565: left oid 39 (ObjNum 1113 snap 324 seq_num 1113) 2024-09-21T10:02:47.094 INFO:tasks.rados.rados.0.smithi033.stdout:3569: finishing write tid 1 to smithi03385152-35 2024-09-21T10:02:47.094 INFO:tasks.rados.rados.0.smithi033.stdout:3565: done (4 left) 2024-09-21T10:02:47.094 INFO:tasks.rados.rados.0.smithi033.stdout:3566: done (3 left) 2024-09-21T10:02:47.094 INFO:tasks.rados.rados.0.smithi033.stdout:3568: done (2 left) 2024-09-21T10:02:47.094 INFO:tasks.rados.rados.0.smithi033.stdout:3571: rollback oid 31 current snap is 324 2024-09-21T10:02:47.094 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 31 to 322 2024-09-21T10:02:47.094 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 48 current snap is 324 2024-09-21T10:02:47.094 INFO:tasks.rados.rados.0.smithi033.stdout:3572: seq_num 1116 ranges {2416640=696320,3112960=573440,3686400=450560,4136960=180224} 2024-09-21T10:02:47.108 INFO:tasks.rados.rados.0.smithi033.stdout:3572: writing smithi03385152-48 from 2416640 to 3112960 tid 1 2024-09-21T10:02:47.111 INFO:tasks.rados.rados.0.smithi033.stdout:3572: writing smithi03385152-48 from 3112960 to 3686400 tid 2 2024-09-21T10:02:47.114 INFO:tasks.rados.rados.0.smithi033.stdout:3572: writing smithi03385152-48 from 3686400 to 4136960 tid 3 2024-09-21T10:02:47.115 INFO:tasks.rados.rados.0.smithi033.stdout:3572: writing smithi03385152-48 from 4136960 to 4317184 tid 4 2024-09-21T10:02:47.115 INFO:tasks.rados.rados.0.smithi033.stdout:3573: delete oid 25 current snap is 324 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3569: finishing write tid 2 to smithi03385152-35 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3570: finishing rollback tid 0 to smithi03385152-47 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3570: finishing rollback tid 1 to smithi03385152-47 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3570: finishing rollback tid 2 to smithi03385152-47 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1518 (ObjNum 1011 snap 298 seq_num 1011) dirty exists 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3569: finishing write tid 3 to smithi03385152-35 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3569: finishing write tid 4 to smithi03385152-35 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3571: finishing rollback tid 0 to smithi03385152-31 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3571: finishing rollback tid 1 to smithi03385152-31 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3571: finishing rollback tid 2 to smithi03385152-31 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2588 (ObjNum 1067 snap 313 seq_num 1067) dirty exists 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3569: finishing write tid 5 to smithi03385152-35 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2585 (ObjNum 1115 snap 324 seq_num 1115) dirty exists 2024-09-21T10:02:47.116 INFO:tasks.rados.rados.0.smithi033.stdout:3569: left oid 35 (ObjNum 1115 snap 324 seq_num 1115) 2024-09-21T10:02:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:3572: finishing write tid 1 to smithi03385152-48 2024-09-21T10:02:47.117 INFO:tasks.rados.rados.0.smithi033.stdout:3572: finishing write tid 2 to smithi03385152-48 2024-09-21T10:02:47.118 INFO:tasks.rados.rados.0.smithi033.stdout:3569: done (4 left) 2024-09-21T10:02:47.118 INFO:tasks.rados.rados.0.smithi033.stdout:3570: done (3 left) 2024-09-21T10:02:47.118 INFO:tasks.rados.rados.0.smithi033.stdout:3571: done (2 left) 2024-09-21T10:02:47.118 INFO:tasks.rados.rados.0.smithi033.stdout:3573: done (1 left) 2024-09-21T10:02:47.118 INFO:tasks.rados.rados.0.smithi033.stdout:3574: read oid 16 snap -1 2024-09-21T10:02:47.118 INFO:tasks.rados.rados.0.smithi033.stdout:3574: expect (ObjNum 1050 snap 308 seq_num 1050) 2024-09-21T10:02:47.118 INFO:tasks.rados.rados.0.smithi033.stdout:3575: delete oid 30 current snap is 324 2024-09-21T10:02:47.119 INFO:tasks.rados.rados.0.smithi033.stdout:3572: finishing write tid 3 to smithi03385152-48 2024-09-21T10:02:47.119 INFO:tasks.rados.rados.0.smithi033.stdout:3572: finishing write tid 4 to smithi03385152-48 2024-09-21T10:02:47.119 INFO:tasks.rados.rados.0.smithi033.stdout:3572: finishing write tid 5 to smithi03385152-48 2024-09-21T10:02:47.125 INFO:tasks.rados.rados.0.smithi033.stdout:3574: expect (ObjNum 1050 snap 308 seq_num 1050) 2024-09-21T10:02:47.166 INFO:tasks.rados.rados.0.smithi033.stdout:3572: finishing write tid 6 to smithi03385152-48 2024-09-21T10:02:47.166 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 48 v 2434 (ObjNum 1116 snap 324 seq_num 1116) dirty exists 2024-09-21T10:02:47.167 INFO:tasks.rados.rados.0.smithi033.stdout:3572: left oid 48 (ObjNum 1116 snap 324 seq_num 1116) 2024-09-21T10:02:47.167 INFO:tasks.rados.rados.0.smithi033.stdout:3572: done (2 left) 2024-09-21T10:02:47.167 INFO:tasks.rados.rados.0.smithi033.stdout:3574: done (1 left) 2024-09-21T10:02:47.167 INFO:tasks.rados.rados.0.smithi033.stdout:3575: done (0 left) 2024-09-21T10:02:47.167 INFO:tasks.rados.rados.0.smithi033.stdout:3576: copy_from oid 16 from oid 14 current snap is 324 2024-09-21T10:02:47.167 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 324 2024-09-21T10:02:47.167 INFO:tasks.rados.rados.0.smithi033.stdout:3577: seq_num 1119 ranges {4923392=745472,5668864=679936,6348800=524288} 2024-09-21T10:02:47.197 INFO:tasks.rados.rados.0.smithi033.stdout:3577: writing smithi03385152-1 from 4923392 to 5668864 tid 1 2024-09-21T10:02:47.201 INFO:tasks.rados.rados.0.smithi033.stdout:3577: writing smithi03385152-1 from 5668864 to 6348800 tid 2 2024-09-21T10:02:47.204 INFO:tasks.rados.rados.0.smithi033.stdout:3577: writing smithi03385152-1 from 6348800 to 6873088 tid 3 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3578: read oid 27 snap -1 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3578: expect (ObjNum 1033 snap 304 seq_num 1033) 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3579: read oid 23 snap 322 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3579: expect (ObjNum 1083 snap 319 seq_num 1083) 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 37 current snap is 324 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3576: finishing copy_from racing read to smithi03385152-16 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3576: finishing copy_from to smithi03385152-16 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2398 (ObjNum 915 snap 265 seq_num 915) dirty exists 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3577: finishing write tid 1 to smithi03385152-1 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3577: finishing write tid 2 to smithi03385152-1 2024-09-21T10:02:47.205 INFO:tasks.rados.rados.0.smithi033.stdout:3580: seq_num 1120 ranges {4997120=761856,5758976=557056,6316032=425984,6742016=221184} 2024-09-21T10:02:47.236 INFO:tasks.rados.rados.0.smithi033.stdout:3580: writing smithi03385152-37 from 4997120 to 5758976 tid 1 2024-09-21T10:02:47.239 INFO:tasks.rados.rados.0.smithi033.stdout:3580: writing smithi03385152-37 from 5758976 to 6316032 tid 2 2024-09-21T10:02:47.241 INFO:tasks.rados.rados.0.smithi033.stdout:3580: writing smithi03385152-37 from 6316032 to 6742016 tid 3 2024-09-21T10:02:47.243 INFO:tasks.rados.rados.0.smithi033.stdout:3580: writing smithi03385152-37 from 6742016 to 6963200 tid 4 2024-09-21T10:02:47.243 INFO:tasks.rados.rados.0.smithi033.stdout:3576: done (4 left) 2024-09-21T10:02:47.243 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 324 2024-09-21T10:02:47.243 INFO:tasks.rados.rados.0.smithi033.stdout:3577: finishing write tid 3 to smithi03385152-1 2024-09-21T10:02:47.243 INFO:tasks.rados.rados.0.smithi033.stdout:3581: seq_num 1121 ranges {0=778240,778240=442368,1220608=638976,1859584=122880} 2024-09-21T10:02:47.247 INFO:tasks.rados.rados.0.smithi033.stdout:3581: writing smithi03385152-44 from 0 to 778240 tid 1 2024-09-21T10:02:47.250 INFO:tasks.rados.rados.0.smithi033.stdout:3581: writing smithi03385152-44 from 778240 to 1220608 tid 2 2024-09-21T10:02:47.253 INFO:tasks.rados.rados.0.smithi033.stdout:3581: writing smithi03385152-44 from 1220608 to 1859584 tid 3 2024-09-21T10:02:47.254 INFO:tasks.rados.rados.0.smithi033.stdout:3581: writing smithi03385152-44 from 1859584 to 1982464 tid 4 2024-09-21T10:02:47.254 INFO:tasks.rados.rados.0.smithi033.stdout:3582: read oid 35 snap -1 2024-09-21T10:02:47.255 INFO:tasks.rados.rados.0.smithi033.stdout:3582: expect (ObjNum 1115 snap 324 seq_num 1115) 2024-09-21T10:02:47.255 INFO:tasks.rados.rados.0.smithi033.stdout:3583: delete oid 16 current snap is 324 2024-09-21T10:02:47.255 INFO:tasks.rados.rados.0.smithi033.stdout:3577: finishing write tid 4 to smithi03385152-1 2024-09-21T10:02:47.255 INFO:tasks.rados.rados.0.smithi033.stdout:3577: finishing write tid 5 to smithi03385152-1 2024-09-21T10:02:47.255 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1638 (ObjNum 1119 snap 324 seq_num 1119) dirty exists 2024-09-21T10:02:47.255 INFO:tasks.rados.rados.0.smithi033.stdout:3577: left oid 1 (ObjNum 1119 snap 324 seq_num 1119) 2024-09-21T10:02:47.255 INFO:tasks.rados.rados.0.smithi033.stdout:3578: expect (ObjNum 1033 snap 304 seq_num 1033) 2024-09-21T10:02:47.275 INFO:tasks.rados.rados.0.smithi033.stdout:3579: expect (ObjNum 1083 snap 319 seq_num 1083) 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3580: finishing write tid 1 to smithi03385152-37 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3580: finishing write tid 2 to smithi03385152-37 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3580: finishing write tid 3 to smithi03385152-37 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3580: finishing write tid 4 to smithi03385152-37 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3580: finishing write tid 5 to smithi03385152-37 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3580: finishing write tid 6 to smithi03385152-37 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 2481 (ObjNum 1120 snap 324 seq_num 1120) dirty exists 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3580: left oid 37 (ObjNum 1120 snap 324 seq_num 1120) 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3581: finishing write tid 1 to smithi03385152-44 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3581: finishing write tid 2 to smithi03385152-44 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3581: finishing write tid 3 to smithi03385152-44 2024-09-21T10:02:47.364 INFO:tasks.rados.rados.0.smithi033.stdout:3577: done (6 left) 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3578: done (5 left) 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3579: done (4 left) 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3580: done (3 left) 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3583: done (2 left) 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 324 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3581: finishing write tid 4 to smithi03385152-44 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3581: finishing write tid 5 to smithi03385152-44 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3581: finishing write tid 6 to smithi03385152-44 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 2386 (ObjNum 1121 snap 324 seq_num 1121) dirty exists 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3581: left oid 44 (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:02:47.365 INFO:tasks.rados.rados.0.smithi033.stdout:3582: expect (ObjNum 1115 snap 324 seq_num 1115) 2024-09-21T10:02:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:47 smithi033 ceph-mon[96568]: pgmap v727: 121 pgs: 121 active+clean; 220 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 6.7 MiB/s wr, 36 op/s 2024-09-21T10:02:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:47 smithi033 ceph-mon[96568]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T10:02:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:47 smithi033 ceph-mon[98546]: pgmap v727: 121 pgs: 121 active+clean; 220 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 6.7 MiB/s wr, 36 op/s 2024-09-21T10:02:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:47 smithi033 ceph-mon[98546]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T10:02:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:47 smithi105 ceph-mon[86649]: pgmap v727: 121 pgs: 121 active+clean; 220 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 6.7 MiB/s wr, 36 op/s 2024-09-21T10:02:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:47 smithi105 ceph-mon[86649]: osdmap e729: 8 total, 8 up, 8 in 2024-09-21T10:02:49.179 INFO:tasks.rados.rados.0.smithi033.stdout:3584: seq_num 1123 ranges {614400=409600,1024000=614400,1638400=598016,2236416=393216} 2024-09-21T10:02:49.185 INFO:tasks.rados.rados.0.smithi033.stdout:3584: writing smithi03385152-27 from 614400 to 1024000 tid 1 2024-09-21T10:02:49.188 INFO:tasks.rados.rados.0.smithi033.stdout:3584: writing smithi03385152-27 from 1024000 to 1638400 tid 2 2024-09-21T10:02:49.191 INFO:tasks.rados.rados.0.smithi033.stdout:3584: writing smithi03385152-27 from 1638400 to 2236416 tid 3 2024-09-21T10:02:49.194 INFO:tasks.rados.rados.0.smithi033.stdout:3584: writing smithi03385152-27 from 2236416 to 2629632 tid 4 2024-09-21T10:02:49.194 INFO:tasks.rados.rados.0.smithi033.stdout:3581: done (2 left) 2024-09-21T10:02:49.194 INFO:tasks.rados.rados.0.smithi033.stdout:3582: done (1 left) 2024-09-21T10:02:49.194 INFO:tasks.rados.rados.0.smithi033.stdout:3585: rollback oid 19 current snap is 324 2024-09-21T10:02:49.195 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 19 to 322 2024-09-21T10:02:49.195 INFO:tasks.rados.rados.0.smithi033.stdout:3584: finishing write tid 1 to smithi03385152-27 2024-09-21T10:02:49.195 INFO:tasks.rados.rados.0.smithi033.stdout:3584: finishing write tid 2 to smithi03385152-27 2024-09-21T10:02:49.195 INFO:tasks.rados.rados.0.smithi033.stdout:3586: rollback oid 24 current snap is 324 2024-09-21T10:02:49.195 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 314 2024-09-21T10:02:49.195 INFO:tasks.rados.rados.0.smithi033.stdout:3587: snap_create 2024-09-21T10:02:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:3585: finishing rollback tid 0 to smithi03385152-19 2024-09-21T10:02:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:3584: finishing write tid 3 to smithi03385152-27 2024-09-21T10:02:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:3585: finishing rollback tid 1 to smithi03385152-19 2024-09-21T10:02:49.197 INFO:tasks.rados.rados.0.smithi033.stdout:3584: finishing write tid 4 to smithi03385152-27 2024-09-21T10:02:49.198 INFO:tasks.rados.rados.0.smithi033.stdout:3584: finishing write tid 5 to smithi03385152-27 2024-09-21T10:02:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:3585: finishing rollback tid 2 to smithi03385152-19 2024-09-21T10:02:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2280 (ObjNum 1082 snap 319 seq_num 1082) dirty exists 2024-09-21T10:02:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:3584: finishing write tid 6 to smithi03385152-27 2024-09-21T10:02:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2480 (ObjNum 1123 snap 324 seq_num 1123) dirty exists 2024-09-21T10:02:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:3584: left oid 27 (ObjNum 1123 snap 324 seq_num 1123) 2024-09-21T10:02:49.199 INFO:tasks.rados.rados.0.smithi033.stdout:3586: finishing rollback tid 1 to smithi03385152-24 2024-09-21T10:02:49.200 INFO:tasks.rados.rados.0.smithi033.stdout:3586: finishing rollback tid 2 to smithi03385152-24 2024-09-21T10:02:49.200 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 2282 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:02:49.677 INFO:tasks.rados.rados.0.smithi033.stdout:3584: done (3 left) 2024-09-21T10:02:49.677 INFO:tasks.rados.rados.0.smithi033.stdout:3585: done (2 left) 2024-09-21T10:02:49.677 INFO:tasks.rados.rados.0.smithi033.stdout:3586: done (1 left) 2024-09-21T10:02:49.678 INFO:tasks.rados.rados.0.smithi033.stdout:3587: done (0 left) 2024-09-21T10:02:49.678 INFO:tasks.rados.rados.0.smithi033.stdout:3588: read oid 49 snap -1 2024-09-21T10:02:49.678 INFO:tasks.rados.rados.0.smithi033.stdout:3588: expect deleted 2024-09-21T10:02:49.678 INFO:tasks.rados.rados.0.smithi033.stdout:3589: snap_remove snap 314 2024-09-21T10:02:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:49 smithi033 ceph-mon[98546]: pgmap v729: 121 pgs: 121 active+clean; 220 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2024-09-21T10:02:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:49 smithi033 ceph-mon[98546]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T10:02:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:49 smithi033 ceph-mon[96568]: pgmap v729: 121 pgs: 121 active+clean; 220 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2024-09-21T10:02:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:49 smithi033 ceph-mon[96568]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T10:02:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:49 smithi105 ceph-mon[86649]: pgmap v729: 121 pgs: 121 active+clean; 220 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 5.5 MiB/s wr, 30 op/s 2024-09-21T10:02:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:49 smithi105 ceph-mon[86649]: osdmap e730: 8 total, 8 up, 8 in 2024-09-21T10:02:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:02:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:02:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:02:50.691 INFO:tasks.rados.rados.0.smithi033.stdout:3589: done (1 left) 2024-09-21T10:02:50.691 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 325 2024-09-21T10:02:50.691 INFO:tasks.rados.rados.0.smithi033.stdout:3590: seq_num 1124 ranges {4816896=434176,5251072=499712,5750784=409600,6160384=663552,6823936=32768} 2024-09-21T10:02:50.720 INFO:tasks.rados.rados.0.smithi033.stdout:3590: writing smithi03385152-12 from 4816896 to 5251072 tid 1 2024-09-21T10:02:50.723 INFO:tasks.rados.rados.0.smithi033.stdout:3590: writing smithi03385152-12 from 5251072 to 5750784 tid 2 2024-09-21T10:02:50.725 INFO:tasks.rados.rados.0.smithi033.stdout:3590: writing smithi03385152-12 from 5750784 to 6160384 tid 3 2024-09-21T10:02:50.729 INFO:tasks.rados.rados.0.smithi033.stdout:3590: writing smithi03385152-12 from 6160384 to 6823936 tid 4 2024-09-21T10:02:50.730 INFO:tasks.rados.rados.0.smithi033.stdout:3590: writing smithi03385152-12 from 6823936 to 6856704 tid 5 2024-09-21T10:02:50.730 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 325 2024-09-21T10:02:50.730 INFO:tasks.rados.rados.0.smithi033.stdout:3591: seq_num 1125 ranges {8814592=450560,9265152=786432,10051584=614400,10665984=204800} 2024-09-21T10:02:50.780 INFO:tasks.rados.rados.0.smithi033.stdout:3591: writing smithi03385152-45 from 8814592 to 9265152 tid 1 2024-09-21T10:02:50.785 INFO:tasks.rados.rados.0.smithi033.stdout:3591: writing smithi03385152-45 from 9265152 to 10051584 tid 2 2024-09-21T10:02:50.788 INFO:tasks.rados.rados.0.smithi033.stdout:3591: writing smithi03385152-45 from 10051584 to 10665984 tid 3 2024-09-21T10:02:50.789 INFO:tasks.rados.rados.0.smithi033.stdout:3591: writing smithi03385152-45 from 10665984 to 10870784 tid 4 2024-09-21T10:02:50.789 INFO:tasks.rados.rados.0.smithi033.stdout:3592: snap_remove snap 322 2024-09-21T10:02:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:50 smithi033 ceph-mon[98546]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T10:02:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:50 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:50 smithi033 ceph-mon[96568]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T10:02:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:50 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:50 smithi105 ceph-mon[86649]: osdmap e731: 8 total, 8 up, 8 in 2024-09-21T10:02:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:50 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:02:51.709 INFO:tasks.rados.rados.0.smithi033.stdout:3592: done (3 left) 2024-09-21T10:02:51.709 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 325 2024-09-21T10:02:51.709 INFO:tasks.rados.rados.0.smithi033.stdout:3593: seq_num 1126 ranges {1572864=466944,2039808=720896,2760704=778240,3538944=106496} 2024-09-21T10:02:51.721 INFO:tasks.rados.rados.0.smithi033.stdout:3593: writing smithi03385152-50 from 1572864 to 2039808 tid 1 2024-09-21T10:02:51.726 INFO:tasks.rados.rados.0.smithi033.stdout:3593: writing smithi03385152-50 from 2039808 to 2760704 tid 2 2024-09-21T10:02:51.733 INFO:tasks.rados.rados.0.smithi033.stdout:3593: writing smithi03385152-50 from 2760704 to 3538944 tid 3 2024-09-21T10:02:51.734 INFO:tasks.rados.rados.0.smithi033.stdout:3593: writing smithi03385152-50 from 3538944 to 3645440 tid 4 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3594: rollback oid 34 current snap is 325 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 324 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3590: finishing write tid 1 to smithi03385152-12 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3590: finishing write tid 2 to smithi03385152-12 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3590: finishing write tid 3 to smithi03385152-12 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3590: finishing write tid 4 to smithi03385152-12 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3590: finishing write tid 5 to smithi03385152-12 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3590: finishing write tid 6 to smithi03385152-12 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3590: finishing write tid 7 to smithi03385152-12 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2484 (ObjNum 1124 snap 325 seq_num 1124) dirty exists 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3590: left oid 12 (ObjNum 1124 snap 325 seq_num 1124) 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3591: finishing write tid 1 to smithi03385152-45 2024-09-21T10:02:51.735 INFO:tasks.rados.rados.0.smithi033.stdout:3588: done (4 left) 2024-09-21T10:02:51.736 INFO:tasks.rados.rados.0.smithi033.stdout:3590: done (3 left) 2024-09-21T10:02:51.736 INFO:tasks.rados.rados.0.smithi033.stdout:3595: rollback oid 5 current snap is 325 2024-09-21T10:02:51.736 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 5 to 324 2024-09-21T10:02:51.736 INFO:tasks.rados.rados.0.smithi033.stdout:3591: finishing write tid 2 to smithi03385152-45 2024-09-21T10:02:51.736 INFO:tasks.rados.rados.0.smithi033.stdout:3591: finishing write tid 3 to smithi03385152-45 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:3591: finishing write tid 4 to smithi03385152-45 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:3591: finishing write tid 5 to smithi03385152-45 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:3591: finishing write tid 6 to smithi03385152-45 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2439 (ObjNum 1125 snap 325 seq_num 1125) dirty exists 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:3591: left oid 45 (ObjNum 1125 snap 325 seq_num 1125) 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:3593: finishing write tid 1 to smithi03385152-50 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:3591: done (3 left) 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 325 2024-09-21T10:02:51.737 INFO:tasks.rados.rados.0.smithi033.stdout:3596: seq_num 1127 ranges {2629632=483328,3112960=598016,3710976=581632,4292608=425984} 2024-09-21T10:02:51.752 INFO:tasks.rados.rados.0.smithi033.stdout:3596: writing smithi03385152-27 from 2629632 to 3112960 tid 1 2024-09-21T10:02:51.754 INFO:tasks.rados.rados.0.smithi033.stdout:3596: writing smithi03385152-27 from 3112960 to 3710976 tid 2 2024-09-21T10:02:51.757 INFO:tasks.rados.rados.0.smithi033.stdout:3596: writing smithi03385152-27 from 3710976 to 4292608 tid 3 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3596: writing smithi03385152-27 from 4292608 to 4718592 tid 4 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3597: copy_from oid 9 from oid 15 current snap is 325 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3595: finishing rollback tid 1 to smithi03385152-5 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 0 (ObjNum 593 snap 0 seq_num 91617144) dirty dne 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3593: finishing write tid 2 to smithi03385152-50 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3593: finishing write tid 3 to smithi03385152-50 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3594: finishing rollback tid 0 to smithi03385152-34 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3594: finishing rollback tid 1 to smithi03385152-34 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3593: finishing write tid 4 to smithi03385152-50 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:3594: finishing rollback tid 2 to smithi03385152-34 2024-09-21T10:02:51.761 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 2442 (ObjNum 1035 snap 305 seq_num 1035) dirty exists 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3593: finishing write tid 5 to smithi03385152-50 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3593: finishing write tid 6 to smithi03385152-50 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 2397 (ObjNum 1126 snap 325 seq_num 1126) dirty exists 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3593: left oid 50 (ObjNum 1126 snap 325 seq_num 1126) 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3596: finishing write tid 1 to smithi03385152-27 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3596: finishing write tid 2 to smithi03385152-27 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3593: done (4 left) 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3594: done (3 left) 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3595: done (2 left) 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3598: copy_from oid 42 from oid 43 current snap is 325 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3599: setattr oid 5 current snap is 325 2024-09-21T10:02:51.762 INFO:tasks.rados.rados.0.smithi033.stdout:3596: finishing write tid 3 to smithi03385152-27 2024-09-21T10:02:51.763 INFO:tasks.rados.rados.0.smithi033.stdout:3598: finishing copy_from to smithi03385152-42 2024-09-21T10:02:51.763 INFO:tasks.rados.rados.0.smithi033.stdout:3598: got expected ENOENT (src dne) 2024-09-21T10:02:51.763 INFO:tasks.rados.rados.0.smithi033.stdout:3598: finishing copy_from racing read to smithi03385152-42 2024-09-21T10:02:51.763 INFO:tasks.rados.rados.0.smithi033.stdout:3598: done (3 left) 2024-09-21T10:02:51.763 INFO:tasks.rados.rados.0.smithi033.stdout:3600: delete oid 49 current snap is 325 2024-09-21T10:02:51.763 INFO:tasks.rados.rados.0.smithi033.stdout:3596: finishing write tid 4 to smithi03385152-27 2024-09-21T10:02:51.764 INFO:tasks.rados.rados.0.smithi033.stdout:3596: finishing write tid 5 to smithi03385152-27 2024-09-21T10:02:51.765 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2413 (ObjNum 593 snap 0 seq_num 91617144) dirty exists 2024-09-21T10:02:51.765 INFO:tasks.rados.rados.0.smithi033.stdout:3599: done (3 left) 2024-09-21T10:02:51.765 INFO:tasks.rados.rados.0.smithi033.stdout:3600: done (2 left) 2024-09-21T10:02:51.765 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 325 2024-09-21T10:02:51.765 INFO:tasks.rados.rados.0.smithi033.stdout:3601: seq_num 1129 ranges {2023424=516096,2539520=770048,3309568=598016,3907584=237568} 2024-09-21T10:02:51.779 INFO:tasks.rados.rados.0.smithi033.stdout:3601: writing smithi03385152-46 from 2023424 to 2539520 tid 1 2024-09-21T10:02:51.783 INFO:tasks.rados.rados.0.smithi033.stdout:3601: writing smithi03385152-46 from 2539520 to 3309568 tid 2 2024-09-21T10:02:51.786 INFO:tasks.rados.rados.0.smithi033.stdout:3601: writing smithi03385152-46 from 3309568 to 3907584 tid 3 2024-09-21T10:02:51.788 INFO:tasks.rados.rados.0.smithi033.stdout:3601: writing smithi03385152-46 from 3907584 to 4145152 tid 4 2024-09-21T10:02:51.788 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 325 2024-09-21T10:02:51.788 INFO:tasks.rados.rados.0.smithi033.stdout:3596: finishing write tid 6 to smithi03385152-27 2024-09-21T10:02:51.788 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2485 (ObjNum 1127 snap 325 seq_num 1127) dirty exists 2024-09-21T10:02:51.788 INFO:tasks.rados.rados.0.smithi033.stdout:3596: left oid 27 (ObjNum 1127 snap 325 seq_num 1127) 2024-09-21T10:02:51.788 INFO:tasks.rados.rados.0.smithi033.stdout:3597: finishing copy_from racing read to smithi03385152-9 2024-09-21T10:02:51.788 INFO:tasks.rados.rados.0.smithi033.stdout:3597: finishing copy_from to smithi03385152-9 2024-09-21T10:02:51.789 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:51.789 INFO:tasks.rados.rados.0.smithi033.stdout:3601: finishing write tid 1 to smithi03385152-46 2024-09-21T10:02:51.789 INFO:tasks.rados.rados.0.smithi033.stdout:3602: seq_num 1130 ranges {0=532480,532480=655360,1187840=753664,1941504=196608} 2024-09-21T10:02:51.790 INFO:tasks.rados.rados.0.smithi033.stdout:3602: writing smithi03385152-5 from 0 to 532480 tid 1 2024-09-21T10:02:51.794 INFO:tasks.rados.rados.0.smithi033.stdout:3602: writing smithi03385152-5 from 532480 to 1187840 tid 2 2024-09-21T10:02:51.799 INFO:tasks.rados.rados.0.smithi033.stdout:3602: writing smithi03385152-5 from 1187840 to 1941504 tid 3 2024-09-21T10:02:51.803 INFO:tasks.rados.rados.0.smithi033.stdout:3602: writing smithi03385152-5 from 1941504 to 2138112 tid 4 2024-09-21T10:02:51.803 INFO:tasks.rados.rados.0.smithi033.stdout:3596: done (3 left) 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3597: done (2 left) 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3603: setattr oid 35 current snap is 325 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3601: finishing write tid 2 to smithi03385152-46 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3601: finishing write tid 3 to smithi03385152-46 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3601: finishing write tid 4 to smithi03385152-46 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3601: finishing write tid 5 to smithi03385152-46 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3602: finishing write tid 1 to smithi03385152-5 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3601: finishing write tid 6 to smithi03385152-46 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2447 (ObjNum 1129 snap 325 seq_num 1129) dirty exists 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3601: left oid 46 (ObjNum 1129 snap 325 seq_num 1129) 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3602: finishing write tid 2 to smithi03385152-5 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3601: done (2 left) 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3604: rollback oid 14 current snap is 325 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 14 to 324 2024-09-21T10:02:51.804 INFO:tasks.rados.rados.0.smithi033.stdout:3605: setattr oid 22 current snap is 325 2024-09-21T10:02:51.805 INFO:tasks.rados.rados.0.smithi033.stdout:3606: snap_create 2024-09-21T10:02:51.805 INFO:tasks.rados.rados.0.smithi033.stdout:3602: finishing write tid 3 to smithi03385152-5 2024-09-21T10:02:51.805 INFO:tasks.rados.rados.0.smithi033.stdout:3602: finishing write tid 4 to smithi03385152-5 2024-09-21T10:02:51.805 INFO:tasks.rados.rados.0.smithi033.stdout:3602: finishing write tid 5 to smithi03385152-5 2024-09-21T10:02:51.805 INFO:tasks.rados.rados.0.smithi033.stdout:3602: finishing write tid 6 to smithi03385152-5 2024-09-21T10:02:51.805 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2418 (ObjNum 1130 snap 325 seq_num 1130) dirty exists 2024-09-21T10:02:51.805 INFO:tasks.rados.rados.0.smithi033.stdout:3602: left oid 5 (ObjNum 1130 snap 325 seq_num 1130) 2024-09-21T10:02:51.805 INFO:tasks.rados.rados.0.smithi033.stdout:3604: finishing rollback tid 0 to smithi03385152-14 2024-09-21T10:02:51.808 INFO:tasks.rados.rados.0.smithi033.stdout:3604: finishing rollback tid 1 to smithi03385152-14 2024-09-21T10:02:51.808 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2613 (ObjNum 1115 snap 324 seq_num 1115) dirty exists 2024-09-21T10:02:51.808 INFO:tasks.rados.rados.0.smithi033.stdout:3604: finishing rollback tid 2 to smithi03385152-14 2024-09-21T10:02:51.808 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2450 (ObjNum 915 snap 265 seq_num 915) dirty exists 2024-09-21T10:02:51.808 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2451 (ObjNum 1104 snap 324 seq_num 1104) dirty exists 2024-09-21T10:02:51.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:02:51] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.51.0" 2024-09-21T10:02:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[98546]: pgmap v732: 121 pgs: 121 active+clean; 223 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T10:02:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[98546]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T10:02:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:51.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:51.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:51.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:51.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:51.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[96568]: pgmap v732: 121 pgs: 121 active+clean; 223 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T10:02:51.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[96568]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T10:02:51.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:51.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:51.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:51.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:51.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:51 smithi105 ceph-mon[86649]: pgmap v732: 121 pgs: 121 active+clean; 223 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.5 MiB/s wr, 13 op/s 2024-09-21T10:02:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:51 smithi105 ceph-mon[86649]: osdmap e732: 8 total, 8 up, 8 in 2024-09-21T10:02:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:52.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:52.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:02:52.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:02:52.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:02:52.713 INFO:tasks.rados.rados.0.smithi033.stdout:3602: done (4 left) 2024-09-21T10:02:52.713 INFO:tasks.rados.rados.0.smithi033.stdout:3603: done (3 left) 2024-09-21T10:02:52.713 INFO:tasks.rados.rados.0.smithi033.stdout:3604: done (2 left) 2024-09-21T10:02:52.714 INFO:tasks.rados.rados.0.smithi033.stdout:3605: done (1 left) 2024-09-21T10:02:52.714 INFO:tasks.rados.rados.0.smithi033.stdout:3606: done (0 left) 2024-09-21T10:02:52.714 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 326 2024-09-21T10:02:52.714 INFO:tasks.rados.rados.0.smithi033.stdout:3607: seq_num 1131 ranges {10870784=548864,11419648=540672,11960320=565248,12525568=425984,12951552=73728} 2024-09-21T10:02:52.775 INFO:tasks.rados.rados.0.smithi033.stdout:3607: writing smithi03385152-45 from 10870784 to 11419648 tid 1 2024-09-21T10:02:52.778 INFO:tasks.rados.rados.0.smithi033.stdout:3607: writing smithi03385152-45 from 11419648 to 11960320 tid 2 2024-09-21T10:02:52.781 INFO:tasks.rados.rados.0.smithi033.stdout:3607: writing smithi03385152-45 from 11960320 to 12525568 tid 3 2024-09-21T10:02:52.784 INFO:tasks.rados.rados.0.smithi033.stdout:3607: writing smithi03385152-45 from 12525568 to 12951552 tid 4 2024-09-21T10:02:52.784 INFO:tasks.rados.rados.0.smithi033.stdout:3607: writing smithi03385152-45 from 12951552 to 13025280 tid 5 2024-09-21T10:02:52.785 INFO:tasks.rados.rados.0.smithi033.stdout:3608: read oid 41 snap 325 2024-09-21T10:02:52.785 INFO:tasks.rados.rados.0.smithi033.stdout:3608: expect deleted 2024-09-21T10:02:52.785 INFO:tasks.rados.rados.0.smithi033.stdout:3609: delete oid 6 current snap is 326 2024-09-21T10:02:52.786 INFO:tasks.rados.rados.0.smithi033.stdout:3607: finishing write tid 1 to smithi03385152-45 2024-09-21T10:02:52.786 INFO:tasks.rados.rados.0.smithi033.stdout:3607: finishing write tid 2 to smithi03385152-45 2024-09-21T10:02:52.790 INFO:tasks.rados.rados.0.smithi033.stdout:3608: done (2 left) 2024-09-21T10:02:52.790 INFO:tasks.rados.rados.0.smithi033.stdout:3609: done (1 left) 2024-09-21T10:02:52.790 INFO:tasks.rados.rados.0.smithi033.stdout:3610: copy_from oid 29 from oid 8 current snap is 326 2024-09-21T10:02:52.790 INFO:tasks.rados.rados.0.smithi033.stdout:3611: setattr oid 15 current snap is 326 2024-09-21T10:02:52.791 INFO:tasks.rados.rados.0.smithi033.stdout:3607: finishing write tid 3 to smithi03385152-45 2024-09-21T10:02:52.791 INFO:tasks.rados.rados.0.smithi033.stdout:3607: finishing write tid 4 to smithi03385152-45 2024-09-21T10:02:52.791 INFO:tasks.rados.rados.0.smithi033.stdout:3612: read oid 38 snap 324 2024-09-21T10:02:52.791 INFO:tasks.rados.rados.0.smithi033.stdout:3612: expect (ObjNum 1012 snap 299 seq_num 1012) 2024-09-21T10:02:52.791 INFO:tasks.rados.rados.0.smithi033.stdout:3613: setattr oid 19 current snap is 326 2024-09-21T10:02:52.792 INFO:tasks.rados.rados.0.smithi033.stdout:3607: finishing write tid 5 to smithi03385152-45 2024-09-21T10:02:52.793 INFO:tasks.rados.rados.0.smithi033.stdout:3614: read oid 37 snap 325 2024-09-21T10:02:52.793 INFO:tasks.rados.rados.0.smithi033.stdout:3614: expect (ObjNum 1120 snap 324 seq_num 1120) 2024-09-21T10:02:52.793 INFO:tasks.rados.rados.0.smithi033.stdout:3615: delete oid 10 current snap is 326 2024-09-21T10:02:52.793 INFO:tasks.rados.rados.0.smithi033.stdout:3607: finishing write tid 6 to smithi03385152-45 2024-09-21T10:02:52.794 INFO:tasks.rados.rados.0.smithi033.stdout:3612: expect (ObjNum 1012 snap 299 seq_num 1012) 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3607: finishing write tid 7 to smithi03385152-45 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2457 (ObjNum 1131 snap 326 seq_num 1131) dirty exists 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3607: left oid 45 (ObjNum 1131 snap 326 seq_num 1131) 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2283 (ObjNum 1082 snap 319 seq_num 1082) dirty exists 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3607: done (6 left) 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3612: done (5 left) 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3613: done (4 left) 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3615: done (3 left) 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3616: read oid 47 snap -1 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3616: expect (ObjNum 1011 snap 298 seq_num 1011) 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:3617: rollback oid 43 current snap is 326 2024-09-21T10:02:52.801 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 43 to 324 2024-09-21T10:02:52.802 INFO:tasks.rados.rados.0.smithi033.stdout:3618: snap_create 2024-09-21T10:02:52.802 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 1639 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:52.803 INFO:tasks.rados.rados.0.smithi033.stdout:3617: finishing rollback tid 1 to smithi03385152-43 2024-09-21T10:02:52.803 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 0 (ObjNum 33 snap 0 seq_num 114156112) dirty dne 2024-09-21T10:02:52.819 INFO:tasks.rados.rados.0.smithi033.stdout:3616: expect (ObjNum 1011 snap 298 seq_num 1011) 2024-09-21T10:02:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:52 smithi033 ceph-mon[98546]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T10:02:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:52 smithi033 ceph-mon[96568]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T10:02:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:52 smithi105 ceph-mon[86649]: osdmap e733: 8 total, 8 up, 8 in 2024-09-21T10:02:53.058 INFO:tasks.rados.rados.0.smithi033.stdout:3610: finishing copy_from racing read to smithi03385152-29 2024-09-21T10:02:53.059 INFO:tasks.rados.rados.0.smithi033.stdout:3614: expect (ObjNum 1120 snap 324 seq_num 1120) 2024-09-21T10:02:53.521 INFO:tasks.rados.rados.0.smithi033.stdout:3610: finishing copy_from to smithi03385152-29 2024-09-21T10:02:53.521 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 1521 (ObjNum 1001 snap 292 seq_num 1001) dirty exists 2024-09-21T10:02:53.566 INFO:tasks.rados.rados.0.smithi033.stdout:3610: done (5 left) 2024-09-21T10:02:53.566 INFO:tasks.rados.rados.0.smithi033.stdout:3611: done (4 left) 2024-09-21T10:02:53.566 INFO:tasks.rados.rados.0.smithi033.stdout:3614: done (3 left) 2024-09-21T10:02:53.566 INFO:tasks.rados.rados.0.smithi033.stdout:3616: done (2 left) 2024-09-21T10:02:53.566 INFO:tasks.rados.rados.0.smithi033.stdout:3617: done (1 left) 2024-09-21T10:02:53.566 INFO:tasks.rados.rados.0.smithi033.stdout:3618: done (0 left) 2024-09-21T10:02:53.566 INFO:tasks.rados.rados.0.smithi033.stdout:3619: rollback oid 24 current snap is 327 2024-09-21T10:02:53.567 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 326 2024-09-21T10:02:53.567 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 327 2024-09-21T10:02:53.567 INFO:tasks.rados.rados.0.smithi033.stdout:3620: seq_num 1134 ranges {6217728=598016,6815744=638976,7454720=737280,8192000=229376} 2024-09-21T10:02:53.603 INFO:tasks.rados.rados.0.smithi033.stdout:3620: writing smithi03385152-22 from 6217728 to 6815744 tid 1 2024-09-21T10:02:53.607 INFO:tasks.rados.rados.0.smithi033.stdout:3620: writing smithi03385152-22 from 6815744 to 7454720 tid 2 2024-09-21T10:02:53.611 INFO:tasks.rados.rados.0.smithi033.stdout:3620: writing smithi03385152-22 from 7454720 to 8192000 tid 3 2024-09-21T10:02:53.612 INFO:tasks.rados.rados.0.smithi033.stdout:3620: writing smithi03385152-22 from 8192000 to 8421376 tid 4 2024-09-21T10:02:53.613 INFO:tasks.rados.rados.0.smithi033.stdout:3621: delete oid 47 current snap is 327 2024-09-21T10:02:53.613 INFO:tasks.rados.rados.0.smithi033.stdout:3619: finishing rollback tid 0 to smithi03385152-24 2024-09-21T10:02:53.613 INFO:tasks.rados.rados.0.smithi033.stdout:3619: finishing rollback tid 1 to smithi03385152-24 2024-09-21T10:02:53.613 INFO:tasks.rados.rados.0.smithi033.stdout:3619: finishing rollback tid 2 to smithi03385152-24 2024-09-21T10:02:53.613 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 2287 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:02:53.613 INFO:tasks.rados.rados.0.smithi033.stdout:3620: finishing write tid 1 to smithi03385152-22 2024-09-21T10:02:53.613 INFO:tasks.rados.rados.0.smithi033.stdout:3620: finishing write tid 2 to smithi03385152-22 2024-09-21T10:02:53.615 INFO:tasks.rados.rados.0.smithi033.stdout:3619: done (2 left) 2024-09-21T10:02:53.615 INFO:tasks.rados.rados.0.smithi033.stdout:3621: done (1 left) 2024-09-21T10:02:53.615 INFO:tasks.rados.rados.0.smithi033.stdout:3622: copy_from oid 42 from oid 30 current snap is 327 2024-09-21T10:02:53.615 INFO:tasks.rados.rados.0.smithi033.stdout:3623: setattr oid 16 current snap is 327 2024-09-21T10:02:53.617 INFO:tasks.rados.rados.0.smithi033.stdout:3620: finishing write tid 3 to smithi03385152-22 2024-09-21T10:02:53.617 INFO:tasks.rados.rados.0.smithi033.stdout:3620: finishing write tid 4 to smithi03385152-22 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:3624: rollback oid 43 current snap is 327 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 43 to 325 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:3625: rmattr oid 25 current snap is 327 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:3625: done (4 left) 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:3626: rollback oid 15 current snap is 327 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 15 to 324 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:3627: copy_from oid 46 from oid 9 current snap is 327 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:3628: rollback oid 21 current snap is 327 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 324 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 327 2024-09-21T10:02:53.618 INFO:tasks.rados.rados.0.smithi033.stdout:3629: seq_num 1136 ranges {7405568=630784,8036352=409600,8445952=753664,9199616=417792,9617408=24576} 2024-09-21T10:02:53.661 INFO:tasks.rados.rados.0.smithi033.stdout:3629: writing smithi03385152-11 from 7405568 to 8036352 tid 1 2024-09-21T10:02:53.664 INFO:tasks.rados.rados.0.smithi033.stdout:3629: writing smithi03385152-11 from 8036352 to 8445952 tid 2 2024-09-21T10:02:53.668 INFO:tasks.rados.rados.0.smithi033.stdout:3629: writing smithi03385152-11 from 8445952 to 9199616 tid 3 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3629: writing smithi03385152-11 from 9199616 to 9617408 tid 4 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3629: writing smithi03385152-11 from 9617408 to 9641984 tid 5 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3630: read oid 23 snap 324 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3630: expect (ObjNum 1083 snap 319 seq_num 1083) 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3622: finishing copy_from to smithi03385152-42 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3622: got expected ENOENT (src dne) 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3631: read oid 44 snap -1 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3631: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3632: delete oid 34 current snap is 327 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3620: finishing write tid 5 to smithi03385152-22 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3622: finishing copy_from racing read to smithi03385152-42 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3620: finishing write tid 6 to smithi03385152-22 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2462 (ObjNum 1134 snap 327 seq_num 1134) dirty exists 2024-09-21T10:02:53.672 INFO:tasks.rados.rados.0.smithi033.stdout:3620: left oid 22 (ObjNum 1134 snap 327 seq_num 1134) 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3624: finishing rollback tid 1 to smithi03385152-43 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 0 (ObjNum 273 snap 0 seq_num 0) dirty dne 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3628: finishing rollback tid 0 to smithi03385152-21 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3626: finishing rollback tid 0 to smithi03385152-15 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3626: finishing rollback tid 1 to smithi03385152-15 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3628: finishing rollback tid 1 to smithi03385152-21 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3626: finishing rollback tid 2 to smithi03385152-15 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 1642 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2616 (ObjNum 33 snap 0 seq_num 91505392) dirty exists 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3628: finishing rollback tid 2 to smithi03385152-21 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 2452 (ObjNum 1098 snap 323 seq_num 1098) dirty exists 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3627: finishing copy_from racing read to smithi03385152-46 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3627: finishing copy_from to smithi03385152-46 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 1624 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3629: finishing write tid 1 to smithi03385152-11 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3629: finishing write tid 2 to smithi03385152-11 2024-09-21T10:02:53.673 INFO:tasks.rados.rados.0.smithi033.stdout:3629: finishing write tid 3 to smithi03385152-11 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3620: done (10 left) 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3622: done (9 left) 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3623: done (8 left) 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3624: done (7 left) 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3626: done (6 left) 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3627: done (5 left) 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3628: done (4 left) 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3632: done (3 left) 2024-09-21T10:02:53.674 INFO:tasks.rados.rados.0.smithi033.stdout:3633: delete oid 30 current snap is 327 2024-09-21T10:02:53.675 INFO:tasks.rados.rados.0.smithi033.stdout:3629: finishing write tid 4 to smithi03385152-11 2024-09-21T10:02:53.676 INFO:tasks.rados.rados.0.smithi033.stdout:3629: finishing write tid 5 to smithi03385152-11 2024-09-21T10:02:53.676 INFO:tasks.rados.rados.0.smithi033.stdout:3629: finishing write tid 6 to smithi03385152-11 2024-09-21T10:02:53.677 INFO:tasks.rados.rados.0.smithi033.stdout:3633: done (3 left) 2024-09-21T10:02:53.677 INFO:tasks.rados.rados.0.smithi033.stdout:3634: read oid 12 snap -1 2024-09-21T10:02:53.678 INFO:tasks.rados.rados.0.smithi033.stdout:3634: expect (ObjNum 1124 snap 325 seq_num 1124) 2024-09-21T10:02:53.678 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 327 2024-09-21T10:02:53.678 INFO:tasks.rados.rados.0.smithi033.stdout:3635: seq_num 1139 ranges {0=679936,679936=458752,1138688=524288,1662976=540672,2203648=81920} 2024-09-21T10:02:53.681 INFO:tasks.rados.rados.0.smithi033.stdout:3635: writing smithi03385152-25 from 0 to 679936 tid 1 2024-09-21T10:02:53.684 INFO:tasks.rados.rados.0.smithi033.stdout:3635: writing smithi03385152-25 from 679936 to 1138688 tid 2 2024-09-21T10:02:53.687 INFO:tasks.rados.rados.0.smithi033.stdout:3635: writing smithi03385152-25 from 1138688 to 1662976 tid 3 2024-09-21T10:02:53.690 INFO:tasks.rados.rados.0.smithi033.stdout:3635: writing smithi03385152-25 from 1662976 to 2203648 tid 4 2024-09-21T10:02:53.691 INFO:tasks.rados.rados.0.smithi033.stdout:3635: writing smithi03385152-25 from 2203648 to 2285568 tid 5 2024-09-21T10:02:53.691 INFO:tasks.rados.rados.0.smithi033.stdout:3636: snap_create 2024-09-21T10:02:53.691 INFO:tasks.rados.rados.0.smithi033.stdout:3629: finishing write tid 7 to smithi03385152-11 2024-09-21T10:02:53.691 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 2487 (ObjNum 1136 snap 327 seq_num 1136) dirty exists 2024-09-21T10:02:53.691 INFO:tasks.rados.rados.0.smithi033.stdout:3629: left oid 11 (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:02:53.691 INFO:tasks.rados.rados.0.smithi033.stdout:3630: expect (ObjNum 1083 snap 319 seq_num 1083) 2024-09-21T10:02:53.783 INFO:tasks.rados.rados.0.smithi033.stdout:3631: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:02:53.836 INFO:tasks.rados.rados.0.smithi033.stdout:3635: finishing write tid 1 to smithi03385152-25 2024-09-21T10:02:53.836 INFO:tasks.rados.rados.0.smithi033.stdout:3635: finishing write tid 2 to smithi03385152-25 2024-09-21T10:02:53.836 INFO:tasks.rados.rados.0.smithi033.stdout:3635: finishing write tid 3 to smithi03385152-25 2024-09-21T10:02:53.836 INFO:tasks.rados.rados.0.smithi033.stdout:3635: finishing write tid 4 to smithi03385152-25 2024-09-21T10:02:53.836 INFO:tasks.rados.rados.0.smithi033.stdout:3635: finishing write tid 5 to smithi03385152-25 2024-09-21T10:02:53.836 INFO:tasks.rados.rados.0.smithi033.stdout:3635: finishing write tid 6 to smithi03385152-25 2024-09-21T10:02:53.836 INFO:tasks.rados.rados.0.smithi033.stdout:3635: finishing write tid 7 to smithi03385152-25 2024-09-21T10:02:53.837 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 2403 (ObjNum 1139 snap 327 seq_num 1139) dirty exists 2024-09-21T10:02:53.837 INFO:tasks.rados.rados.0.smithi033.stdout:3635: left oid 25 (ObjNum 1139 snap 327 seq_num 1139) 2024-09-21T10:02:53.837 INFO:tasks.rados.rados.0.smithi033.stdout:3634: expect (ObjNum 1124 snap 325 seq_num 1124) 2024-09-21T10:02:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:53 smithi033 ceph-mon[98546]: pgmap v735: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 6.7 MiB/s wr, 25 op/s 2024-09-21T10:02:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:53 smithi033 ceph-mon[98546]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T10:02:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:53 smithi033 ceph-mon[98546]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T10:02:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:53 smithi033 ceph-mon[96568]: pgmap v735: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 6.7 MiB/s wr, 25 op/s 2024-09-21T10:02:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:53 smithi033 ceph-mon[96568]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T10:02:53.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:53 smithi033 ceph-mon[96568]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T10:02:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:53 smithi105 ceph-mon[86649]: pgmap v735: 121 pgs: 121 active+clean; 249 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s rd, 6.7 MiB/s wr, 25 op/s 2024-09-21T10:02:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:53 smithi105 ceph-mon[86649]: osdmap e734: 8 total, 8 up, 8 in 2024-09-21T10:02:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:53 smithi105 ceph-mon[86649]: osdmap e735: 8 total, 8 up, 8 in 2024-09-21T10:02:54.567 INFO:tasks.rados.rados.0.smithi033.stdout:3629: done (5 left) 2024-09-21T10:02:54.567 INFO:tasks.rados.rados.0.smithi033.stdout:3630: done (4 left) 2024-09-21T10:02:54.567 INFO:tasks.rados.rados.0.smithi033.stdout:3631: done (3 left) 2024-09-21T10:02:54.567 INFO:tasks.rados.rados.0.smithi033.stdout:3634: done (2 left) 2024-09-21T10:02:54.567 INFO:tasks.rados.rados.0.smithi033.stdout:3635: done (1 left) 2024-09-21T10:02:54.567 INFO:tasks.rados.rados.0.smithi033.stdout:3636: done (0 left) 2024-09-21T10:02:54.567 INFO:tasks.rados.rados.0.smithi033.stdout:3637: delete oid 25 current snap is 328 2024-09-21T10:02:54.570 INFO:tasks.rados.rados.0.smithi033.stdout:3637: done (0 left) 2024-09-21T10:02:54.570 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 45 current snap is 328 2024-09-21T10:02:54.570 INFO:tasks.rados.rados.0.smithi033.stdout:3638: seq_num 1141 ranges {13025280=712704,13737984=679936,14417920=540672,14958592=385024} 2024-09-21T10:02:54.645 INFO:tasks.rados.rados.0.smithi033.stdout:3638: writing smithi03385152-45 from 13025280 to 13737984 tid 1 2024-09-21T10:02:54.648 INFO:tasks.rados.rados.0.smithi033.stdout:3638: writing smithi03385152-45 from 13737984 to 14417920 tid 2 2024-09-21T10:02:54.652 INFO:tasks.rados.rados.0.smithi033.stdout:3638: writing smithi03385152-45 from 14417920 to 14958592 tid 3 2024-09-21T10:02:54.654 INFO:tasks.rados.rados.0.smithi033.stdout:3638: writing smithi03385152-45 from 14958592 to 15343616 tid 4 2024-09-21T10:02:54.654 INFO:tasks.rados.rados.0.smithi033.stdout:3639: read oid 17 snap -1 2024-09-21T10:02:54.654 INFO:tasks.rados.rados.0.smithi033.stdout:3639: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:02:54.654 INFO:tasks.rados.rados.0.smithi033.stdout:3640: delete oid 32 current snap is 328 2024-09-21T10:02:54.654 INFO:tasks.rados.rados.0.smithi033.stdout:3638: finishing write tid 1 to smithi03385152-45 2024-09-21T10:02:54.655 INFO:tasks.rados.rados.0.smithi033.stdout:3638: finishing write tid 2 to smithi03385152-45 2024-09-21T10:02:54.656 INFO:tasks.rados.rados.0.smithi033.stdout:3640: done (2 left) 2024-09-21T10:02:54.656 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 328 2024-09-21T10:02:54.656 INFO:tasks.rados.rados.0.smithi033.stdout:3641: seq_num 1143 ranges {6873088=753664,7626752=450560,8077312=507904,8585216=589824,9175040=57344} 2024-09-21T10:02:54.698 INFO:tasks.rados.rados.0.smithi033.stdout:3641: writing smithi03385152-1 from 6873088 to 7626752 tid 1 2024-09-21T10:02:54.700 INFO:tasks.rados.rados.0.smithi033.stdout:3641: writing smithi03385152-1 from 7626752 to 8077312 tid 2 2024-09-21T10:02:54.703 INFO:tasks.rados.rados.0.smithi033.stdout:3641: writing smithi03385152-1 from 8077312 to 8585216 tid 3 2024-09-21T10:02:54.706 INFO:tasks.rados.rados.0.smithi033.stdout:3641: writing smithi03385152-1 from 8585216 to 9175040 tid 4 2024-09-21T10:02:54.707 INFO:tasks.rados.rados.0.smithi033.stdout:3641: writing smithi03385152-1 from 9175040 to 9232384 tid 5 2024-09-21T10:02:54.707 INFO:tasks.rados.rados.0.smithi033.stdout:3642: snap_create 2024-09-21T10:02:54.707 INFO:tasks.rados.rados.0.smithi033.stdout:3638: finishing write tid 3 to smithi03385152-45 2024-09-21T10:02:54.707 INFO:tasks.rados.rados.0.smithi033.stdout:3638: finishing write tid 4 to smithi03385152-45 2024-09-21T10:02:54.708 INFO:tasks.rados.rados.0.smithi033.stdout:3638: finishing write tid 5 to smithi03385152-45 2024-09-21T10:02:54.708 INFO:tasks.rados.rados.0.smithi033.stdout:3638: finishing write tid 6 to smithi03385152-45 2024-09-21T10:02:54.708 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2468 (ObjNum 1141 snap 328 seq_num 1141) dirty exists 2024-09-21T10:02:54.708 INFO:tasks.rados.rados.0.smithi033.stdout:3638: left oid 45 (ObjNum 1141 snap 328 seq_num 1141) 2024-09-21T10:02:54.708 INFO:tasks.rados.rados.0.smithi033.stdout:3639: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:02:54.752 INFO:tasks.rados.rados.0.smithi033.stdout:3641: finishing write tid 1 to smithi03385152-1 2024-09-21T10:02:54.752 INFO:tasks.rados.rados.0.smithi033.stdout:3641: finishing write tid 2 to smithi03385152-1 2024-09-21T10:02:54.752 INFO:tasks.rados.rados.0.smithi033.stdout:3641: finishing write tid 3 to smithi03385152-1 2024-09-21T10:02:54.752 INFO:tasks.rados.rados.0.smithi033.stdout:3641: finishing write tid 4 to smithi03385152-1 2024-09-21T10:02:54.752 INFO:tasks.rados.rados.0.smithi033.stdout:3641: finishing write tid 5 to smithi03385152-1 2024-09-21T10:02:54.752 INFO:tasks.rados.rados.0.smithi033.stdout:3641: finishing write tid 6 to smithi03385152-1 2024-09-21T10:02:54.752 INFO:tasks.rados.rados.0.smithi033.stdout:3641: finishing write tid 7 to smithi03385152-1 2024-09-21T10:02:54.753 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 1648 (ObjNum 1143 snap 328 seq_num 1143) dirty exists 2024-09-21T10:02:54.753 INFO:tasks.rados.rados.0.smithi033.stdout:3641: left oid 1 (ObjNum 1143 snap 328 seq_num 1143) 2024-09-21T10:02:55.580 INFO:tasks.rados.rados.0.smithi033.stdout:3638: done (3 left) 2024-09-21T10:02:55.580 INFO:tasks.rados.rados.0.smithi033.stdout:3639: done (2 left) 2024-09-21T10:02:55.580 INFO:tasks.rados.rados.0.smithi033.stdout:3641: done (1 left) 2024-09-21T10:02:55.580 INFO:tasks.rados.rados.0.smithi033.stdout:3642: done (0 left) 2024-09-21T10:02:55.580 INFO:tasks.rados.rados.0.smithi033.stdout:3643: delete oid 26 current snap is 329 2024-09-21T10:02:55.585 INFO:tasks.rados.rados.0.smithi033.stdout:3643: done (0 left) 2024-09-21T10:02:55.585 INFO:tasks.rados.rados.0.smithi033.stdout:3644: copy_from oid 40 from oid 42 current snap is 329 2024-09-21T10:02:55.585 INFO:tasks.rados.rados.0.smithi033.stdout:3645: rmattr oid 28 current snap is 329 2024-09-21T10:02:55.585 INFO:tasks.rados.rados.0.smithi033.stdout:3646: delete oid 50 current snap is 329 2024-09-21T10:02:55.589 INFO:tasks.rados.rados.0.smithi033.stdout:3646: done (2 left) 2024-09-21T10:02:55.589 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 329 2024-09-21T10:02:55.589 INFO:tasks.rados.rados.0.smithi033.stdout:3647: seq_num 1146 ranges {0=802816,802816=499712,1302528=679936,1982464=425984} 2024-09-21T10:02:55.595 INFO:tasks.rados.rados.0.smithi033.stdout:3647: writing smithi03385152-6 from 0 to 802816 tid 1 2024-09-21T10:02:55.598 INFO:tasks.rados.rados.0.smithi033.stdout:3647: writing smithi03385152-6 from 802816 to 1302528 tid 2 2024-09-21T10:02:55.602 INFO:tasks.rados.rados.0.smithi033.stdout:3647: writing smithi03385152-6 from 1302528 to 1982464 tid 3 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:3647: writing smithi03385152-6 from 1982464 to 2408448 tid 4 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:3648: read oid 44 snap -1 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:3648: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:3649: read oid 30 snap -1 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:3649: expect deleted 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 329 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1835 (ObjNum 1090 snap 321 seq_num 1090) dirty exists 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:3644: finishing copy_from racing read to smithi03385152-40 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:3644: finishing copy_from to smithi03385152-40 2024-09-21T10:02:55.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 2409 (ObjNum 1103 snap 324 seq_num 1103) dirty exists 2024-09-21T10:02:55.606 INFO:tasks.rados.rados.0.smithi033.stdout:3647: finishing write tid 1 to smithi03385152-6 2024-09-21T10:02:55.606 INFO:tasks.rados.rados.0.smithi033.stdout:3647: finishing write tid 2 to smithi03385152-6 2024-09-21T10:02:55.606 INFO:tasks.rados.rados.0.smithi033.stdout:3650: seq_num 1147 ranges {2023424=16384} 2024-09-21T10:02:55.616 INFO:tasks.rados.rados.0.smithi033.stdout:3650: writing smithi03385152-46 from 2023424 to 2039808 tid 1 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3644: done (5 left) 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3645: done (4 left) 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3651: rollback oid 34 current snap is 329 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 34 to 325 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3647: finishing write tid 3 to smithi03385152-6 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3649: done (4 left) 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3652: rollback oid 17 current snap is 329 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 327 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3647: finishing write tid 4 to smithi03385152-6 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3647: finishing write tid 5 to smithi03385152-6 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3647: finishing write tid 6 to smithi03385152-6 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 2491 (ObjNum 1146 snap 329 seq_num 1146) dirty exists 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3647: left oid 6 (ObjNum 1146 snap 329 seq_num 1146) 2024-09-21T10:02:55.617 INFO:tasks.rados.rados.0.smithi033.stdout:3648: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3652: finishing rollback tid 0 to smithi03385152-17 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3650: finishing write tid 1 to smithi03385152-46 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3652: finishing rollback tid 1 to smithi03385152-17 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3652: finishing rollback tid 2 to smithi03385152-17 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2488 (ObjNum 1103 snap 324 seq_num 1103) dirty exists 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3650: finishing write tid 2 to smithi03385152-46 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3647: done (4 left) 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3648: done (3 left) 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3652: done (2 left) 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3653: snap_create 2024-09-21T10:02:55.669 INFO:tasks.rados.rados.0.smithi033.stdout:3651: finishing rollback tid 1 to smithi03385152-34 2024-09-21T10:02:55.670 INFO:tasks.rados.rados.0.smithi033.stdout:3651: finishing rollback tid 2 to smithi03385152-34 2024-09-21T10:02:55.670 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 2472 (ObjNum 1035 snap 305 seq_num 1035) dirty exists 2024-09-21T10:02:55.670 INFO:tasks.rados.rados.0.smithi033.stdout:3650: finishing write tid 3 to smithi03385152-46 2024-09-21T10:02:55.670 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2470 (ObjNum 1147 snap 329 seq_num 1147) dirty exists 2024-09-21T10:02:55.670 INFO:tasks.rados.rados.0.smithi033.stdout:3650: left oid 46 (ObjNum 1147 snap 329 seq_num 1147) 2024-09-21T10:02:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:55 smithi033 ceph-mon[98546]: pgmap v738: 121 pgs: 121 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 32 op/s 2024-09-21T10:02:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:55 smithi033 ceph-mon[98546]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T10:02:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:55 smithi033 ceph-mon[96568]: pgmap v738: 121 pgs: 121 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 32 op/s 2024-09-21T10:02:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:55 smithi033 ceph-mon[96568]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T10:02:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:55 smithi105 ceph-mon[86649]: pgmap v738: 121 pgs: 121 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 22 MiB/s rd, 6.2 MiB/s wr, 32 op/s 2024-09-21T10:02:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:55 smithi105 ceph-mon[86649]: osdmap e736: 8 total, 8 up, 8 in 2024-09-21T10:02:56.586 INFO:tasks.rados.rados.0.smithi033.stdout:3650: done (2 left) 2024-09-21T10:02:56.586 INFO:tasks.rados.rados.0.smithi033.stdout:3651: done (1 left) 2024-09-21T10:02:56.586 INFO:tasks.rados.rados.0.smithi033.stdout:3653: done (0 left) 2024-09-21T10:02:56.586 INFO:tasks.rados.rados.0.smithi033.stdout:3654: read oid 15 snap -1 2024-09-21T10:02:56.586 INFO:tasks.rados.rados.0.smithi033.stdout:3654: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:56.586 INFO:tasks.rados.rados.0.smithi033.stdout:3655: read oid 49 snap -1 2024-09-21T10:02:56.587 INFO:tasks.rados.rados.0.smithi033.stdout:3655: expect deleted 2024-09-21T10:02:56.587 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 330 2024-09-21T10:02:56.587 INFO:tasks.rados.rados.0.smithi033.stdout:3656: seq_num 1148 ranges {2867200=32768} 2024-09-21T10:02:56.602 INFO:tasks.rados.rados.0.smithi033.stdout:3656: writing smithi03385152-21 from 2867200 to 2899968 tid 1 2024-09-21T10:02:56.602 INFO:tasks.rados.rados.0.smithi033.stdout:3657: copy_from oid 8 from oid 46 current snap is 330 2024-09-21T10:02:56.602 INFO:tasks.rados.rados.0.smithi033.stdout:3655: done (3 left) 2024-09-21T10:02:56.602 INFO:tasks.rados.rados.0.smithi033.stdout:3658: copy_from oid 14 from oid 29 current snap is 330 2024-09-21T10:02:56.602 INFO:tasks.rados.rados.0.smithi033.stdout:3654: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:02:56.658 INFO:tasks.rados.rados.0.smithi033.stdout:3656: finishing write tid 1 to smithi03385152-21 2024-09-21T10:02:56.658 INFO:tasks.rados.rados.0.smithi033.stdout:3656: finishing write tid 2 to smithi03385152-21 2024-09-21T10:02:56.658 INFO:tasks.rados.rados.0.smithi033.stdout:3656: finishing write tid 3 to smithi03385152-21 2024-09-21T10:02:56.658 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 2454 (ObjNum 1148 snap 330 seq_num 1148) dirty exists 2024-09-21T10:02:56.658 INFO:tasks.rados.rados.0.smithi033.stdout:3656: left oid 21 (ObjNum 1148 snap 330 seq_num 1148) 2024-09-21T10:02:56.658 INFO:tasks.rados.rados.0.smithi033.stdout:3657: finishing copy_from racing read to smithi03385152-8 2024-09-21T10:02:56.658 INFO:tasks.rados.rados.0.smithi033.stdout:3657: finishing copy_from to smithi03385152-8 2024-09-21T10:02:56.658 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 2470 (ObjNum 1147 snap 329 seq_num 1147) dirty exists 2024-09-21T10:02:56.659 INFO:tasks.rados.rados.0.smithi033.stdout:3658: finishing copy_from racing read to smithi03385152-14 2024-09-21T10:02:56.659 INFO:tasks.rados.rados.0.smithi033.stdout:3654: done (3 left) 2024-09-21T10:02:56.659 INFO:tasks.rados.rados.0.smithi033.stdout:3656: done (2 left) 2024-09-21T10:02:56.659 INFO:tasks.rados.rados.0.smithi033.stdout:3657: done (1 left) 2024-09-21T10:02:56.659 INFO:tasks.rados.rados.0.smithi033.stdout:3659: delete oid 45 current snap is 330 2024-09-21T10:02:56.659 INFO:tasks.rados.rados.0.smithi033.stdout:3658: finishing copy_from to smithi03385152-14 2024-09-21T10:02:56.659 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 1521 (ObjNum 1001 snap 292 seq_num 1001) dirty exists 2024-09-21T10:02:56.661 INFO:tasks.rados.rados.0.smithi033.stdout:3658: done (1 left) 2024-09-21T10:02:56.661 INFO:tasks.rados.rados.0.smithi033.stdout:3659: done (0 left) 2024-09-21T10:02:56.661 INFO:tasks.rados.rados.0.smithi033.stdout:3660: read oid 24 snap -1 2024-09-21T10:02:56.661 INFO:tasks.rados.rados.0.smithi033.stdout:3660: expect (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:02:56.661 INFO:tasks.rados.rados.0.smithi033.stdout:3661: delete oid 38 current snap is 330 2024-09-21T10:02:56.666 INFO:tasks.rados.rados.0.smithi033.stdout:3661: done (1 left) 2024-09-21T10:02:56.666 INFO:tasks.rados.rados.0.smithi033.stdout:3662: rollback oid 21 current snap is 330 2024-09-21T10:02:56.666 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 21 to 325 2024-09-21T10:02:56.666 INFO:tasks.rados.rados.0.smithi033.stdout:3663: read oid 43 snap -1 2024-09-21T10:02:56.666 INFO:tasks.rados.rados.0.smithi033.stdout:3663: expect deleted 2024-09-21T10:02:56.666 INFO:tasks.rados.rados.0.smithi033.stdout:3664: read oid 38 snap -1 2024-09-21T10:02:56.666 INFO:tasks.rados.rados.0.smithi033.stdout:3664: expect deleted 2024-09-21T10:02:56.666 INFO:tasks.rados.rados.0.smithi033.stdout:3665: snap_remove snap 328 2024-09-21T10:02:56.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:56 smithi033 ceph-mon[98546]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T10:02:56.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:56 smithi033 ceph-mon[96568]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T10:02:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:56 smithi105 ceph-mon[86649]: osdmap e737: 8 total, 8 up, 8 in 2024-09-21T10:02:57.597 INFO:tasks.rados.rados.0.smithi033.stdout:3660: expect (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:02:57.655 INFO:tasks.rados.rados.0.smithi033.stdout:3662: finishing rollback tid 0 to smithi03385152-21 2024-09-21T10:02:57.655 INFO:tasks.rados.rados.0.smithi033.stdout:3662: finishing rollback tid 1 to smithi03385152-21 2024-09-21T10:02:57.655 INFO:tasks.rados.rados.0.smithi033.stdout:3662: finishing rollback tid 2 to smithi03385152-21 2024-09-21T10:02:57.655 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 2457 (ObjNum 1098 snap 323 seq_num 1098) dirty exists 2024-09-21T10:02:57.655 INFO:tasks.rados.rados.0.smithi033.stdout:3660: done (4 left) 2024-09-21T10:02:57.658 INFO:tasks.rados.rados.0.smithi033.stdout:3662: done (3 left) 2024-09-21T10:02:57.658 INFO:tasks.rados.rados.0.smithi033.stdout:3663: done (2 left) 2024-09-21T10:02:57.658 INFO:tasks.rados.rados.0.smithi033.stdout:3664: done (1 left) 2024-09-21T10:02:57.658 INFO:tasks.rados.rados.0.smithi033.stdout:3665: done (0 left) 2024-09-21T10:02:57.658 INFO:tasks.rados.rados.0.smithi033.stdout:3666: read oid 40 snap 329 2024-09-21T10:02:57.658 INFO:tasks.rados.rados.0.smithi033.stdout:3666: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:02:57.658 INFO:tasks.rados.rados.0.smithi033.stdout:3667: read oid 12 snap 324 2024-09-21T10:02:57.658 INFO:tasks.rados.rados.0.smithi033.stdout:3667: expect (ObjNum 1072 snap 314 seq_num 1072) 2024-09-21T10:02:57.659 INFO:tasks.rados.rados.0.smithi033.stdout:3668: rollback oid 45 current snap is 330 2024-09-21T10:02:57.659 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 45 to 326 2024-09-21T10:02:57.659 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 330 2024-09-21T10:02:57.659 INFO:tasks.rados.rados.0.smithi033.stdout:3669: seq_num 1151 ranges {9232384=81920} 2024-09-21T10:02:57.707 INFO:tasks.rados.rados.0.smithi033.stdout:3669: writing smithi03385152-1 from 9232384 to 9314304 tid 1 2024-09-21T10:02:57.707 INFO:tasks.rados.rados.0.smithi033.stdout:3670: read oid 34 snap -1 2024-09-21T10:02:57.707 INFO:tasks.rados.rados.0.smithi033.stdout:3670: expect (ObjNum 1035 snap 305 seq_num 1035) 2024-09-21T10:02:57.707 INFO:tasks.rados.rados.0.smithi033.stdout:3671: rollback oid 24 current snap is 330 2024-09-21T10:02:57.708 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 24 to 325 2024-09-21T10:02:57.708 INFO:tasks.rados.rados.0.smithi033.stdout:3668: finishing rollback tid 1 to smithi03385152-45 2024-09-21T10:02:57.708 INFO:tasks.rados.rados.0.smithi033.stdout:3668: finishing rollback tid 2 to smithi03385152-45 2024-09-21T10:02:57.708 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2475 (ObjNum 1131 snap 326 seq_num 1131) dirty exists 2024-09-21T10:02:57.708 INFO:tasks.rados.rados.0.smithi033.stdout:3666: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:02:57.752 INFO:tasks.rados.rados.0.smithi033.stdout:3667: expect (ObjNum 1072 snap 314 seq_num 1072) 2024-09-21T10:02:57.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:57 smithi033 ceph-mon[98546]: pgmap v741: 121 pgs: 121 active+clean; 203 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.6 MiB/s wr, 56 op/s 2024-09-21T10:02:57.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:57 smithi033 ceph-mon[98546]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T10:02:57.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:57 smithi033 ceph-mon[96568]: pgmap v741: 121 pgs: 121 active+clean; 203 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.6 MiB/s wr, 56 op/s 2024-09-21T10:02:57.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:57 smithi033 ceph-mon[96568]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T10:02:58.019 INFO:tasks.rados.rados.0.smithi033.stdout:3669: finishing write tid 1 to smithi03385152-1 2024-09-21T10:02:58.019 INFO:tasks.rados.rados.0.smithi033.stdout:3671: finishing rollback tid 0 to smithi03385152-24 2024-09-21T10:02:58.019 INFO:tasks.rados.rados.0.smithi033.stdout:3671: finishing rollback tid 1 to smithi03385152-24 2024-09-21T10:02:58.019 INFO:tasks.rados.rados.0.smithi033.stdout:3666: done (5 left) 2024-09-21T10:02:58.020 INFO:tasks.rados.rados.0.smithi033.stdout:3667: done (4 left) 2024-09-21T10:02:58.020 INFO:tasks.rados.rados.0.smithi033.stdout:3668: done (3 left) 2024-09-21T10:02:58.020 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 50 current snap is 330 2024-09-21T10:02:58.020 INFO:tasks.rados.rados.0.smithi033.stdout:3669: finishing write tid 2 to smithi03385152-1 2024-09-21T10:02:58.020 INFO:tasks.rados.rados.0.smithi033.stdout:3671: finishing rollback tid 2 to smithi03385152-24 2024-09-21T10:02:58.020 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 2290 (ObjNum 910 snap 263 seq_num 910) dirty exists 2024-09-21T10:02:58.020 INFO:tasks.rados.rados.0.smithi033.stdout:3670: expect (ObjNum 1035 snap 305 seq_num 1035) 2024-09-21T10:02:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:57 smithi105 ceph-mon[86649]: pgmap v741: 121 pgs: 121 active+clean; 203 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 8.6 MiB/s wr, 56 op/s 2024-09-21T10:02:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:57 smithi105 ceph-mon[86649]: osdmap e738: 8 total, 8 up, 8 in 2024-09-21T10:02:58.040 INFO:tasks.rados.rados.0.smithi033.stdout:3669: finishing write tid 3 to smithi03385152-1 2024-09-21T10:02:58.040 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 2472 (ObjNum 1151 snap 330 seq_num 1151) dirty exists 2024-09-21T10:02:58.040 INFO:tasks.rados.rados.0.smithi033.stdout:3669: left oid 1 (ObjNum 1151 snap 330 seq_num 1151) 2024-09-21T10:02:58.040 INFO:tasks.rados.rados.0.smithi033.stdout:3672: seq_num 1152 ranges {0=98304} 2024-09-21T10:02:58.040 INFO:tasks.rados.rados.0.smithi033.stdout:3672: writing smithi03385152-50 from 0 to 98304 tid 1 2024-09-21T10:02:58.040 INFO:tasks.rados.rados.0.smithi033.stdout:3669: done (3 left) 2024-09-21T10:02:58.041 INFO:tasks.rados.rados.0.smithi033.stdout:3670: done (2 left) 2024-09-21T10:02:58.041 INFO:tasks.rados.rados.0.smithi033.stdout:3671: done (1 left) 2024-09-21T10:02:58.041 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 330 2024-09-21T10:02:58.041 INFO:tasks.rados.rados.0.smithi033.stdout:3673: seq_num 1153 ranges {0=114688} 2024-09-21T10:02:58.041 INFO:tasks.rados.rados.0.smithi033.stdout:3673: writing smithi03385152-3 from 0 to 114688 tid 1 2024-09-21T10:02:58.041 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 330 2024-09-21T10:02:58.041 INFO:tasks.rados.rados.0.smithi033.stdout:3674: seq_num 1154 ranges {2039808=131072} 2024-09-21T10:02:58.053 INFO:tasks.rados.rados.0.smithi033.stdout:3674: writing smithi03385152-46 from 2039808 to 2170880 tid 1 2024-09-21T10:02:58.053 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 330 2024-09-21T10:02:58.053 INFO:tasks.rados.rados.0.smithi033.stdout:3672: finishing write tid 1 to smithi03385152-50 2024-09-21T10:02:58.053 INFO:tasks.rados.rados.0.smithi033.stdout:3672: finishing write tid 2 to smithi03385152-50 2024-09-21T10:02:58.053 INFO:tasks.rados.rados.0.smithi033.stdout:3673: finishing write tid 1 to smithi03385152-3 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:3673: finishing write tid 2 to smithi03385152-3 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:3672: finishing write tid 3 to smithi03385152-50 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 50 v 2407 (ObjNum 1152 snap 330 seq_num 1152) dirty exists 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:3672: left oid 50 (ObjNum 1152 snap 330 seq_num 1152) 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:3673: finishing write tid 3 to smithi03385152-3 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2477 (ObjNum 1153 snap 330 seq_num 1153) dirty exists 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:3673: left oid 3 (ObjNum 1153 snap 330 seq_num 1153) 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:3675: seq_num 1155 ranges {0=147456} 2024-09-21T10:02:58.054 INFO:tasks.rados.rados.0.smithi033.stdout:3675: writing smithi03385152-38 from 0 to 147456 tid 1 2024-09-21T10:02:58.055 INFO:tasks.rados.rados.0.smithi033.stdout:3672: done (3 left) 2024-09-21T10:02:58.055 INFO:tasks.rados.rados.0.smithi033.stdout:3673: done (2 left) 2024-09-21T10:02:58.055 INFO:tasks.rados.rados.0.smithi033.stdout:3676: snap_remove snap 324 2024-09-21T10:02:58.568 INFO:tasks.rados.rados.0.smithi033.stdout:3676: done (2 left) 2024-09-21T10:02:58.568 INFO:tasks.rados.rados.0.smithi033.stdout:3677: read oid 20 snap -1 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3677: expect deleted 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3678: snap_create 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3675: finishing write tid 1 to smithi03385152-38 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3675: finishing write tid 2 to smithi03385152-38 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3674: finishing write tid 1 to smithi03385152-46 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3674: finishing write tid 2 to smithi03385152-46 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3675: finishing write tid 3 to smithi03385152-38 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2414 (ObjNum 1155 snap 330 seq_num 1155) dirty exists 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3675: left oid 38 (ObjNum 1155 snap 330 seq_num 1155) 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3674: finishing write tid 3 to smithi03385152-46 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2479 (ObjNum 1154 snap 330 seq_num 1154) dirty exists 2024-09-21T10:02:58.569 INFO:tasks.rados.rados.0.smithi033.stdout:3674: left oid 46 (ObjNum 1154 snap 330 seq_num 1154) 2024-09-21T10:02:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:58 smithi033 ceph-mon[96568]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T10:02:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:58 smithi033 ceph-mon[96568]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T10:02:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:58 smithi033 ceph-mon[98546]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T10:02:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:58 smithi033 ceph-mon[98546]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T10:02:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:58 smithi105 ceph-mon[86649]: osdmap e739: 8 total, 8 up, 8 in 2024-09-21T10:02:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:58 smithi105 ceph-mon[86649]: osdmap e740: 8 total, 8 up, 8 in 2024-09-21T10:02:59.572 INFO:tasks.rados.rados.0.smithi033.stdout:3674: done (3 left) 2024-09-21T10:02:59.572 INFO:tasks.rados.rados.0.smithi033.stdout:3675: done (2 left) 2024-09-21T10:02:59.572 INFO:tasks.rados.rados.0.smithi033.stdout:3677: done (1 left) 2024-09-21T10:02:59.572 INFO:tasks.rados.rados.0.smithi033.stdout:3678: done (0 left) 2024-09-21T10:02:59.572 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 331 2024-09-21T10:02:59.572 INFO:tasks.rados.rados.0.smithi033.stdout:3679: seq_num 1156 ranges {1785856=163840} 2024-09-21T10:02:59.583 INFO:tasks.rados.rados.0.smithi033.stdout:3679: writing smithi03385152-18 from 1785856 to 1949696 tid 1 2024-09-21T10:02:59.583 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 331 2024-09-21T10:02:59.583 INFO:tasks.rados.rados.0.smithi033.stdout:3680: seq_num 1157 ranges {2138112=180224} 2024-09-21T10:02:59.596 INFO:tasks.rados.rados.0.smithi033.stdout:3680: writing smithi03385152-5 from 2138112 to 2318336 tid 1 2024-09-21T10:02:59.596 INFO:tasks.rados.rados.0.smithi033.stdout:3681: delete oid 35 current snap is 331 2024-09-21T10:02:59.596 INFO:tasks.rados.rados.0.smithi033.stdout:3679: finishing write tid 1 to smithi03385152-18 2024-09-21T10:02:59.596 INFO:tasks.rados.rados.0.smithi033.stdout:3679: finishing write tid 2 to smithi03385152-18 2024-09-21T10:02:59.596 INFO:tasks.rados.rados.0.smithi033.stdout:3679: finishing write tid 3 to smithi03385152-18 2024-09-21T10:02:59.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1830 (ObjNum 1156 snap 331 seq_num 1156) dirty exists 2024-09-21T10:02:59.596 INFO:tasks.rados.rados.0.smithi033.stdout:3679: left oid 18 (ObjNum 1156 snap 331 seq_num 1156) 2024-09-21T10:02:59.601 INFO:tasks.rados.rados.0.smithi033.stdout:3679: done (2 left) 2024-09-21T10:02:59.601 INFO:tasks.rados.rados.0.smithi033.stdout:3681: done (1 left) 2024-09-21T10:02:59.601 INFO:tasks.rados.rados.0.smithi033.stdout:3682: delete oid 33 current snap is 331 2024-09-21T10:02:59.604 INFO:tasks.rados.rados.0.smithi033.stdout:3680: finishing write tid 1 to smithi03385152-5 2024-09-21T10:02:59.604 INFO:tasks.rados.rados.0.smithi033.stdout:3680: finishing write tid 2 to smithi03385152-5 2024-09-21T10:02:59.605 INFO:tasks.rados.rados.0.smithi033.stdout:3680: finishing write tid 3 to smithi03385152-5 2024-09-21T10:02:59.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2420 (ObjNum 1157 snap 331 seq_num 1157) dirty exists 2024-09-21T10:02:59.605 INFO:tasks.rados.rados.0.smithi033.stdout:3680: left oid 5 (ObjNum 1157 snap 331 seq_num 1157) 2024-09-21T10:02:59.605 INFO:tasks.rados.rados.0.smithi033.stdout:3680: done (1 left) 2024-09-21T10:02:59.605 INFO:tasks.rados.rados.0.smithi033.stdout:3682: done (0 left) 2024-09-21T10:02:59.605 INFO:tasks.rados.rados.0.smithi033.stdout:3683: snap_create 2024-09-21T10:02:59.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:59 smithi105 ceph-mon[86649]: pgmap v744: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.7 MiB/s wr, 46 op/s 2024-09-21T10:02:59.932 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:02:59 smithi105 ceph-mon[86649]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T10:02:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:59 smithi033 ceph-mon[96568]: pgmap v744: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.7 MiB/s wr, 46 op/s 2024-09-21T10:02:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:02:59 smithi033 ceph-mon[96568]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T10:02:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:59 smithi033 ceph-mon[98546]: pgmap v744: 121 pgs: 1 active+clean+snaptrim, 120 active+clean; 202 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 6.7 MiB/s wr, 46 op/s 2024-09-21T10:02:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:02:59 smithi033 ceph-mon[98546]: osdmap e741: 8 total, 8 up, 8 in 2024-09-21T10:03:00.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:03:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:03:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:00.606 INFO:tasks.rados.rados.0.smithi033.stdout:3683: done (0 left) 2024-09-21T10:03:00.606 INFO:tasks.rados.rados.0.smithi033.stdout:3684: copy_from oid 23 from oid 22 current snap is 332 2024-09-21T10:03:00.606 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 332 2024-09-21T10:03:00.606 INFO:tasks.rados.rados.0.smithi033.stdout:3685: seq_num 1160 ranges {0=229376} 2024-09-21T10:03:00.608 INFO:tasks.rados.rados.0.smithi033.stdout:3685: writing smithi03385152-33 from 0 to 229376 tid 1 2024-09-21T10:03:00.608 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 332 2024-09-21T10:03:00.608 INFO:tasks.rados.rados.0.smithi033.stdout:3686: seq_num 1161 ranges {0=245760} 2024-09-21T10:03:00.609 INFO:tasks.rados.rados.0.smithi033.stdout:3686: writing smithi03385152-13 from 0 to 245760 tid 1 2024-09-21T10:03:00.609 INFO:tasks.rados.rados.0.smithi033.stdout:3687: rmattr oid 16 current snap is 332 2024-09-21T10:03:00.609 INFO:tasks.rados.rados.0.smithi033.stdout:3688: snap_create 2024-09-21T10:03:00.616 INFO:tasks.rados.rados.0.smithi033.stdout:3685: finishing write tid 1 to smithi03385152-33 2024-09-21T10:03:00.616 INFO:tasks.rados.rados.0.smithi033.stdout:3685: finishing write tid 2 to smithi03385152-33 2024-09-21T10:03:00.617 INFO:tasks.rados.rados.0.smithi033.stdout:3686: finishing write tid 1 to smithi03385152-13 2024-09-21T10:03:00.617 INFO:tasks.rados.rados.0.smithi033.stdout:3686: finishing write tid 2 to smithi03385152-13 2024-09-21T10:03:00.617 INFO:tasks.rados.rados.0.smithi033.stdout:3685: finishing write tid 3 to smithi03385152-33 2024-09-21T10:03:00.617 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 2494 (ObjNum 1160 snap 332 seq_num 1160) dirty exists 2024-09-21T10:03:00.617 INFO:tasks.rados.rados.0.smithi033.stdout:3685: left oid 33 (ObjNum 1160 snap 332 seq_num 1160) 2024-09-21T10:03:00.618 INFO:tasks.rados.rados.0.smithi033.stdout:3686: finishing write tid 3 to smithi03385152-13 2024-09-21T10:03:00.618 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2625 (ObjNum 1161 snap 332 seq_num 1161) dirty exists 2024-09-21T10:03:00.618 INFO:tasks.rados.rados.0.smithi033.stdout:3686: left oid 13 (ObjNum 1161 snap 332 seq_num 1161) 2024-09-21T10:03:00.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2627 (ObjNum 49 snap 0 seq_num 1953066355) dirty exists 2024-09-21T10:03:00.649 INFO:tasks.rados.rados.0.smithi033.stdout:3684: finishing copy_from racing read to smithi03385152-23 2024-09-21T10:03:00.653 INFO:tasks.rados.rados.0.smithi033.stdout:3684: finishing copy_from to smithi03385152-23 2024-09-21T10:03:00.653 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 2462 (ObjNum 1134 snap 327 seq_num 1134) dirty exists 2024-09-21T10:03:01.608 INFO:tasks.rados.rados.0.smithi033.stdout:3684: done (4 left) 2024-09-21T10:03:01.608 INFO:tasks.rados.rados.0.smithi033.stdout:3685: done (3 left) 2024-09-21T10:03:01.608 INFO:tasks.rados.rados.0.smithi033.stdout:3686: done (2 left) 2024-09-21T10:03:01.608 INFO:tasks.rados.rados.0.smithi033.stdout:3687: done (1 left) 2024-09-21T10:03:01.608 INFO:tasks.rados.rados.0.smithi033.stdout:3688: done (0 left) 2024-09-21T10:03:01.608 INFO:tasks.rados.rados.0.smithi033.stdout:3689: delete oid 7 current snap is 333 2024-09-21T10:03:01.615 INFO:tasks.rados.rados.0.smithi033.stdout:3689: done (0 left) 2024-09-21T10:03:01.615 INFO:tasks.rados.rados.0.smithi033.stdout:3690: snap_remove snap 326 2024-09-21T10:03:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:03:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:03:01] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-09-21T10:03:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:01 smithi033 ceph-mon[98546]: pgmap v747: 121 pgs: 3 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 115 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-09-21T10:03:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:01 smithi033 ceph-mon[98546]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T10:03:01.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:01 smithi033 ceph-mon[96568]: pgmap v747: 121 pgs: 3 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 115 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-09-21T10:03:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:01 smithi033 ceph-mon[96568]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T10:03:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:01 smithi105 ceph-mon[86649]: pgmap v747: 121 pgs: 3 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 115 active+clean; 206 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-09-21T10:03:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:01 smithi105 ceph-mon[86649]: osdmap e742: 8 total, 8 up, 8 in 2024-09-21T10:03:02.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:02.633 INFO:tasks.rados.rados.0.smithi033.stdout:3690: done (0 left) 2024-09-21T10:03:02.633 INFO:tasks.rados.rados.0.smithi033.stdout:3691: read oid 20 snap -1 2024-09-21T10:03:02.633 INFO:tasks.rados.rados.0.smithi033.stdout:3691: expect deleted 2024-09-21T10:03:02.633 INFO:tasks.rados.rados.0.smithi033.stdout:3692: read oid 12 snap -1 2024-09-21T10:03:02.633 INFO:tasks.rados.rados.0.smithi033.stdout:3692: expect (ObjNum 1124 snap 325 seq_num 1124) 2024-09-21T10:03:02.634 INFO:tasks.rados.rados.0.smithi033.stdout:3693: read oid 6 snap -1 2024-09-21T10:03:02.634 INFO:tasks.rados.rados.0.smithi033.stdout:3693: expect (ObjNum 1146 snap 329 seq_num 1146) 2024-09-21T10:03:02.634 INFO:tasks.rados.rados.0.smithi033.stdout:3694: read oid 17 snap -1 2024-09-21T10:03:02.634 INFO:tasks.rados.rados.0.smithi033.stdout:3694: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:03:02.636 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 333 2024-09-21T10:03:02.636 INFO:tasks.rados.rados.0.smithi033.stdout:3695: seq_num 1163 ranges {0=286720} 2024-09-21T10:03:02.636 INFO:tasks.rados.rados.0.smithi033.stdout:3695: writing smithi03385152-16 from 0 to 286720 tid 1 2024-09-21T10:03:02.636 INFO:tasks.rados.rados.0.smithi033.stdout:3696: read oid 30 snap -1 2024-09-21T10:03:02.636 INFO:tasks.rados.rados.0.smithi033.stdout:3696: expect deleted 2024-09-21T10:03:02.637 INFO:tasks.rados.rados.0.smithi033.stdout:3697: delete oid 37 current snap is 333 2024-09-21T10:03:02.644 INFO:tasks.rados.rados.0.smithi033.stdout:3691: done (6 left) 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3697: done (5 left) 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3698: read oid 46 snap 331 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3698: expect (ObjNum 1154 snap 330 seq_num 1154) 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3699: read oid 13 snap -1 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3699: expect (ObjNum 1161 snap 332 seq_num 1161) 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3700: read oid 24 snap -1 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3700: expect (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3701: rollback oid 38 current snap is 333 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 330 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 333 2024-09-21T10:03:02.645 INFO:tasks.rados.rados.0.smithi033.stdout:3702: seq_num 1165 ranges {0=319488} 2024-09-21T10:03:02.646 INFO:tasks.rados.rados.0.smithi033.stdout:3702: writing smithi03385152-7 from 0 to 319488 tid 1 2024-09-21T10:03:02.647 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 333 2024-09-21T10:03:02.647 INFO:tasks.rados.rados.0.smithi033.stdout:3703: seq_num 1166 ranges {0=335872} 2024-09-21T10:03:02.649 INFO:tasks.rados.rados.0.smithi033.stdout:3703: writing smithi03385152-20 from 0 to 335872 tid 1 2024-09-21T10:03:02.649 INFO:tasks.rados.rados.0.smithi033.stdout:3704: read oid 26 snap 325 2024-09-21T10:03:02.649 INFO:tasks.rados.rados.0.smithi033.stdout:3704: expect (ObjNum 1102 snap 324 seq_num 1102) 2024-09-21T10:03:02.649 INFO:tasks.rados.rados.0.smithi033.stdout:3705: snap_remove snap 327 2024-09-21T10:03:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:02 smithi033 ceph-mon[98546]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T10:03:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:02 smithi033 ceph-mon[96568]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T10:03:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:02 smithi105 ceph-mon[86649]: osdmap e743: 8 total, 8 up, 8 in 2024-09-21T10:03:03.572 INFO:tasks.rados.rados.0.smithi033.stdout:3695: finishing write tid 1 to smithi03385152-16 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:3701: finishing rollback tid 0 to smithi03385152-38 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:3701: finishing rollback tid 1 to smithi03385152-38 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:3701: finishing rollback tid 2 to smithi03385152-38 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2417 (ObjNum 1155 snap 330 seq_num 1155) dirty exists 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:3703: finishing write tid 1 to smithi03385152-20 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:3703: finishing write tid 2 to smithi03385152-20 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:3703: finishing write tid 3 to smithi03385152-20 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 2464 (ObjNum 1166 snap 333 seq_num 1166) dirty exists 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:3703: left oid 20 (ObjNum 1166 snap 333 seq_num 1166) 2024-09-21T10:03:03.573 INFO:tasks.rados.rados.0.smithi033.stdout:3700: expect (ObjNum 910 snap 263 seq_num 910) 2024-09-21T10:03:03.635 INFO:tasks.rados.rados.0.smithi033.stdout:3695: finishing write tid 2 to smithi03385152-16 2024-09-21T10:03:03.635 INFO:tasks.rados.rados.0.smithi033.stdout:3693: expect (ObjNum 1146 snap 329 seq_num 1146) 2024-09-21T10:03:03.698 INFO:tasks.rados.rados.0.smithi033.stdout:3698: expect (ObjNum 1154 snap 330 seq_num 1154) 2024-09-21T10:03:03.822 INFO:tasks.rados.rados.0.smithi033.stdout:3702: finishing write tid 1 to smithi03385152-7 2024-09-21T10:03:03.822 INFO:tasks.rados.rados.0.smithi033.stdout:3699: expect (ObjNum 1161 snap 332 seq_num 1161) 2024-09-21T10:03:03.829 INFO:tasks.rados.rados.0.smithi033.stdout:3695: finishing write tid 3 to smithi03385152-16 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2636 (ObjNum 1163 snap 333 seq_num 1163) dirty exists 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3695: left oid 16 (ObjNum 1163 snap 333 seq_num 1163) 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3693: done (12 left) 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3695: done (11 left) 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3696: done (10 left) 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3698: done (9 left) 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3699: done (8 left) 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3700: done (7 left) 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3701: done (6 left) 2024-09-21T10:03:03.830 INFO:tasks.rados.rados.0.smithi033.stdout:3703: done (5 left) 2024-09-21T10:03:03.831 INFO:tasks.rados.rados.0.smithi033.stdout:3705: done (4 left) 2024-09-21T10:03:03.831 INFO:tasks.rados.rados.0.smithi033.stdout:3706: read oid 50 snap 329 2024-09-21T10:03:03.831 INFO:tasks.rados.rados.0.smithi033.stdout:3706: expect deleted 2024-09-21T10:03:03.831 INFO:tasks.rados.rados.0.smithi033.stdout:3707: read oid 20 snap -1 2024-09-21T10:03:03.831 INFO:tasks.rados.rados.0.smithi033.stdout:3707: expect (ObjNum 1166 snap 333 seq_num 1166) 2024-09-21T10:03:03.831 INFO:tasks.rados.rados.0.smithi033.stdout:3708: snap_remove snap 332 2024-09-21T10:03:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:03 smithi033 ceph-mon[98546]: pgmap v750: 121 pgs: 3 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 115 active+clean; 227 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T10:03:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:03 smithi033 ceph-mon[98546]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T10:03:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:03 smithi033 ceph-mon[98546]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T10:03:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:03 smithi033 ceph-mon[96568]: pgmap v750: 121 pgs: 3 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 115 active+clean; 227 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T10:03:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:03 smithi033 ceph-mon[96568]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T10:03:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:03 smithi033 ceph-mon[96568]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T10:03:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:03 smithi105 ceph-mon[86649]: pgmap v750: 121 pgs: 3 active+clean+snaptrim_wait, 3 active+clean+snaptrim, 115 active+clean; 227 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.2 MiB/s wr, 22 op/s 2024-09-21T10:03:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:03 smithi105 ceph-mon[86649]: osdmap e744: 8 total, 8 up, 8 in 2024-09-21T10:03:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:03 smithi105 ceph-mon[86649]: osdmap e745: 8 total, 8 up, 8 in 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:3708: done (6 left) 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:3709: rmattr oid 47 current snap is 333 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:3709: done (6 left) 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:3710: snap_create 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:3702: finishing write tid 2 to smithi03385152-7 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:3702: finishing write tid 3 to smithi03385152-7 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2638 (ObjNum 1165 snap 333 seq_num 1165) dirty exists 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:3702: left oid 7 (ObjNum 1165 snap 333 seq_num 1165) 2024-09-21T10:03:04.632 INFO:tasks.rados.rados.0.smithi033.stdout:3692: expect (ObjNum 1124 snap 325 seq_num 1124) 2024-09-21T10:03:05.034 INFO:tasks.rados.rados.0.smithi033.stdout:3694: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:03:05.078 INFO:tasks.rados.rados.0.smithi033.stdout:3704: expect (ObjNum 1102 snap 324 seq_num 1102) 2024-09-21T10:03:05.122 INFO:tasks.rados.rados.0.smithi033.stdout:3707: expect (ObjNum 1166 snap 333 seq_num 1166) 2024-09-21T10:03:05.634 INFO:tasks.rados.rados.0.smithi033.stdout:3692: done (6 left) 2024-09-21T10:03:05.634 INFO:tasks.rados.rados.0.smithi033.stdout:3694: done (5 left) 2024-09-21T10:03:05.634 INFO:tasks.rados.rados.0.smithi033.stdout:3702: done (4 left) 2024-09-21T10:03:05.634 INFO:tasks.rados.rados.0.smithi033.stdout:3704: done (3 left) 2024-09-21T10:03:05.634 INFO:tasks.rados.rados.0.smithi033.stdout:3706: done (2 left) 2024-09-21T10:03:05.634 INFO:tasks.rados.rados.0.smithi033.stdout:3707: done (1 left) 2024-09-21T10:03:05.634 INFO:tasks.rados.rados.0.smithi033.stdout:3710: done (0 left) 2024-09-21T10:03:05.634 INFO:tasks.rados.rados.0.smithi033.stdout:3711: delete oid 43 current snap is 334 2024-09-21T10:03:05.638 INFO:tasks.rados.rados.0.smithi033.stdout:3711: done (0 left) 2024-09-21T10:03:05.639 INFO:tasks.rados.rados.0.smithi033.stdout:3712: snap_create 2024-09-21T10:03:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:05 smithi033 ceph-mon[98546]: pgmap v753: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 226 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.3 MiB/s wr, 18 op/s 2024-09-21T10:03:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:05 smithi033 ceph-mon[98546]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T10:03:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:05 smithi033 ceph-mon[96568]: pgmap v753: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 226 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.3 MiB/s wr, 18 op/s 2024-09-21T10:03:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:05 smithi033 ceph-mon[96568]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T10:03:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:05 smithi105 ceph-mon[86649]: pgmap v753: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 226 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.3 MiB/s wr, 18 op/s 2024-09-21T10:03:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:05 smithi105 ceph-mon[86649]: osdmap e746: 8 total, 8 up, 8 in 2024-09-21T10:03:06.645 INFO:tasks.rados.rados.0.smithi033.stdout:3712: done (0 left) 2024-09-21T10:03:06.645 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 33 current snap is 335 2024-09-21T10:03:06.646 INFO:tasks.rados.rados.0.smithi033.stdout:3713: seq_num 1168 ranges {229376=368640} 2024-09-21T10:03:06.649 INFO:tasks.rados.rados.0.smithi033.stdout:3713: writing smithi03385152-33 from 229376 to 598016 tid 1 2024-09-21T10:03:06.649 INFO:tasks.rados.rados.0.smithi033.stdout:3714: copy_from oid 10 from oid 40 current snap is 335 2024-09-21T10:03:06.649 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 335 2024-09-21T10:03:06.649 INFO:tasks.rados.rados.0.smithi033.stdout:3715: seq_num 1169 ranges {1949696=385024} 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:3715: writing smithi03385152-18 from 1949696 to 2334720 tid 1 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:3716: setattr oid 34 current snap is 335 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:3713: finishing write tid 1 to smithi03385152-33 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:3713: finishing write tid 2 to smithi03385152-33 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:3713: finishing write tid 3 to smithi03385152-33 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 2496 (ObjNum 1168 snap 335 seq_num 1168) dirty exists 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:3713: left oid 33 (ObjNum 1168 snap 335 seq_num 1168) 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:3714: finishing copy_from racing read to smithi03385152-10 2024-09-21T10:03:06.663 INFO:tasks.rados.rados.0.smithi033.stdout:3714: finishing copy_from to smithi03385152-10 2024-09-21T10:03:06.664 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 2409 (ObjNum 1103 snap 324 seq_num 1103) dirty exists 2024-09-21T10:03:06.664 INFO:tasks.rados.rados.0.smithi033.stdout:3713: done (3 left) 2024-09-21T10:03:06.664 INFO:tasks.rados.rados.0.smithi033.stdout:3714: done (2 left) 2024-09-21T10:03:06.664 INFO:tasks.rados.rados.0.smithi033.stdout:3717: snap_remove snap 334 2024-09-21T10:03:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:06 smithi033 ceph-mon[98546]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T10:03:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:06 smithi033 ceph-mon[96568]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T10:03:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:06 smithi105 ceph-mon[86649]: osdmap e747: 8 total, 8 up, 8 in 2024-09-21T10:03:07.651 INFO:tasks.rados.rados.0.smithi033.stdout:3717: done (2 left) 2024-09-21T10:03:07.652 INFO:tasks.rados.rados.0.smithi033.stdout:3718: snap_remove snap 330 2024-09-21T10:03:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:07 smithi033 ceph-mon[96568]: pgmap v756: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 232 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T10:03:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:07 smithi033 ceph-mon[96568]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T10:03:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:07 smithi033 ceph-mon[98546]: pgmap v756: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 232 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T10:03:07.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:07 smithi033 ceph-mon[98546]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T10:03:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:07 smithi105 ceph-mon[86649]: pgmap v756: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 232 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 2.4 MiB/s wr, 14 op/s 2024-09-21T10:03:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:07 smithi105 ceph-mon[86649]: osdmap e748: 8 total, 8 up, 8 in 2024-09-21T10:03:08.575 INFO:tasks.rados.rados.0.smithi033.stdout:3718: done (2 left) 2024-09-21T10:03:08.575 INFO:tasks.rados.rados.0.smithi033.stdout:3719: snap_remove snap 329 2024-09-21T10:03:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:08 smithi033 ceph-mon[98546]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T10:03:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:08 smithi033 ceph-mon[98546]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T10:03:08.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:08 smithi033 ceph-mon[96568]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T10:03:08.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:08 smithi033 ceph-mon[96568]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T10:03:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:08 smithi105 ceph-mon[86649]: osdmap e749: 8 total, 8 up, 8 in 2024-09-21T10:03:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:08 smithi105 ceph-mon[86649]: osdmap e750: 8 total, 8 up, 8 in 2024-09-21T10:03:09.574 INFO:tasks.rados.rados.0.smithi033.stdout:3719: done (2 left) 2024-09-21T10:03:09.574 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 49 current snap is 335 2024-09-21T10:03:09.574 INFO:tasks.rados.rados.0.smithi033.stdout:3720: seq_num 1170 ranges {0=401408} 2024-09-21T10:03:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3720: writing smithi03385152-49 from 0 to 401408 tid 1 2024-09-21T10:03:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3721: snap_create 2024-09-21T10:03:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3715: finishing write tid 1 to smithi03385152-18 2024-09-21T10:03:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3715: finishing write tid 2 to smithi03385152-18 2024-09-21T10:03:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3715: finishing write tid 3 to smithi03385152-18 2024-09-21T10:03:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1832 (ObjNum 1169 snap 335 seq_num 1169) dirty exists 2024-09-21T10:03:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:3715: left oid 18 (ObjNum 1169 snap 335 seq_num 1169) 2024-09-21T10:03:09.577 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 2480 (ObjNum 1035 snap 305 seq_num 1035) dirty exists 2024-09-21T10:03:09.587 INFO:tasks.rados.rados.0.smithi033.stdout:3720: finishing write tid 1 to smithi03385152-49 2024-09-21T10:03:09.588 INFO:tasks.rados.rados.0.smithi033.stdout:3720: finishing write tid 2 to smithi03385152-49 2024-09-21T10:03:09.590 INFO:tasks.rados.rados.0.smithi033.stdout:3720: finishing write tid 3 to smithi03385152-49 2024-09-21T10:03:09.590 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 49 v 2490 (ObjNum 1170 snap 335 seq_num 1170) dirty exists 2024-09-21T10:03:09.590 INFO:tasks.rados.rados.0.smithi033.stdout:3720: left oid 49 (ObjNum 1170 snap 335 seq_num 1170) 2024-09-21T10:03:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:09 smithi033 ceph-mon[98546]: pgmap v759: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 233 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.3 MiB/s wr, 7 op/s 2024-09-21T10:03:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:09 smithi033 ceph-mon[98546]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T10:03:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:09 smithi033 ceph-mon[96568]: pgmap v759: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 233 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.3 MiB/s wr, 7 op/s 2024-09-21T10:03:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:09 smithi033 ceph-mon[96568]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T10:03:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:09 smithi105 ceph-mon[86649]: pgmap v759: 121 pgs: 2 active+clean+snaptrim, 2 active+clean+snaptrim_wait, 117 active+clean; 233 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.3 MiB/s wr, 7 op/s 2024-09-21T10:03:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:09 smithi105 ceph-mon[86649]: osdmap e751: 8 total, 8 up, 8 in 2024-09-21T10:03:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:03:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:03:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3715: done (3 left) 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3716: done (2 left) 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3720: done (1 left) 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3721: done (0 left) 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3722: rmattr oid 43 current snap is 336 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3722: done (0 left) 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3723: read oid 26 snap 325 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3723: expect (ObjNum 1102 snap 324 seq_num 1102) 2024-09-21T10:03:10.577 INFO:tasks.rados.rados.0.smithi033.stdout:3724: delete oid 33 current snap is 336 2024-09-21T10:03:10.585 INFO:tasks.rados.rados.0.smithi033.stdout:3724: done (1 left) 2024-09-21T10:03:10.585 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 23 current snap is 336 2024-09-21T10:03:10.585 INFO:tasks.rados.rados.0.smithi033.stdout:3725: seq_num 1172 ranges {8421376=434176} 2024-09-21T10:03:10.635 INFO:tasks.rados.rados.0.smithi033.stdout:3725: writing smithi03385152-23 from 8421376 to 8855552 tid 1 2024-09-21T10:03:10.635 INFO:tasks.rados.rados.0.smithi033.stdout:3726: copy_from oid 8 from oid 30 current snap is 336 2024-09-21T10:03:10.635 INFO:tasks.rados.rados.0.smithi033.stdout:3723: expect (ObjNum 1102 snap 324 seq_num 1102) 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3726: finishing copy_from to smithi03385152-8 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3726: got expected ENOENT (src dne) 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3726: finishing copy_from racing read to smithi03385152-8 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3725: finishing write tid 1 to smithi03385152-23 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3725: finishing write tid 2 to smithi03385152-23 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3723: done (2 left) 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3726: done (1 left) 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3727: delete oid 1 current snap is 336 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:3725: finishing write tid 3 to smithi03385152-23 2024-09-21T10:03:10.681 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 2466 (ObjNum 1172 snap 336 seq_num 1172) dirty exists 2024-09-21T10:03:10.682 INFO:tasks.rados.rados.0.smithi033.stdout:3725: left oid 23 (ObjNum 1172 snap 336 seq_num 1172) 2024-09-21T10:03:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3725: done (1 left) 2024-09-21T10:03:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3727: done (0 left) 2024-09-21T10:03:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3728: read oid 12 snap 333 2024-09-21T10:03:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3728: expect (ObjNum 1124 snap 325 seq_num 1124) 2024-09-21T10:03:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 336 2024-09-21T10:03:10.685 INFO:tasks.rados.rados.0.smithi033.stdout:3729: seq_num 1174 ranges {1679360=466944} 2024-09-21T10:03:10.696 INFO:tasks.rados.rados.0.smithi033.stdout:3729: writing smithi03385152-40 from 1679360 to 2146304 tid 1 2024-09-21T10:03:10.697 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 336 2024-09-21T10:03:10.697 INFO:tasks.rados.rados.0.smithi033.stdout:3730: seq_num 1175 ranges {2023424=483328} 2024-09-21T10:03:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3730: writing smithi03385152-9 from 2023424 to 2506752 tid 1 2024-09-21T10:03:10.712 INFO:tasks.rados.rados.0.smithi033.stdout:3731: rollback oid 3 current snap is 336 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 3 to 331 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3729: finishing write tid 1 to smithi03385152-40 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3729: finishing write tid 2 to smithi03385152-40 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3729: finishing write tid 3 to smithi03385152-40 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 2419 (ObjNum 1174 snap 336 seq_num 1174) dirty exists 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3729: left oid 40 (ObjNum 1174 snap 336 seq_num 1174) 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3729: done (3 left) 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3732: copy_from oid 24 from oid 17 current snap is 336 2024-09-21T10:03:10.713 INFO:tasks.rados.rados.0.smithi033.stdout:3733: snap_create 2024-09-21T10:03:10.715 INFO:tasks.rados.rados.0.smithi033.stdout:3728: expect (ObjNum 1124 snap 325 seq_num 1124) 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:3730: finishing write tid 1 to smithi03385152-9 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:3730: finishing write tid 2 to smithi03385152-9 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:3731: finishing rollback tid 0 to smithi03385152-3 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:3731: finishing rollback tid 1 to smithi03385152-3 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:3731: finishing rollback tid 2 to smithi03385152-3 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2483 (ObjNum 1153 snap 330 seq_num 1153) dirty exists 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:3730: finishing write tid 3 to smithi03385152-9 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 2421 (ObjNum 1175 snap 336 seq_num 1175) dirty exists 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:3730: left oid 9 (ObjNum 1175 snap 336 seq_num 1175) 2024-09-21T10:03:11.116 INFO:tasks.rados.rados.0.smithi033.stdout:3732: finishing copy_from racing read to smithi03385152-24 2024-09-21T10:03:11.117 INFO:tasks.rados.rados.0.smithi033.stdout:3732: finishing copy_from to smithi03385152-24 2024-09-21T10:03:11.117 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 24 v 2488 (ObjNum 1103 snap 324 seq_num 1103) dirty exists 2024-09-21T10:03:11.669 INFO:tasks.rados.rados.0.smithi033.stdout:3728: done (4 left) 2024-09-21T10:03:11.669 INFO:tasks.rados.rados.0.smithi033.stdout:3730: done (3 left) 2024-09-21T10:03:11.669 INFO:tasks.rados.rados.0.smithi033.stdout:3731: done (2 left) 2024-09-21T10:03:11.669 INFO:tasks.rados.rados.0.smithi033.stdout:3732: done (1 left) 2024-09-21T10:03:11.669 INFO:tasks.rados.rados.0.smithi033.stdout:3733: done (0 left) 2024-09-21T10:03:11.669 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 337 2024-09-21T10:03:11.670 INFO:tasks.rados.rados.0.smithi033.stdout:3734: seq_num 1176 ranges {8232960=499712} 2024-09-21T10:03:11.717 INFO:tasks.rados.rados.0.smithi033.stdout:3734: writing smithi03385152-2 from 8232960 to 8732672 tid 1 2024-09-21T10:03:11.717 INFO:tasks.rados.rados.0.smithi033.stdout:3735: setattr oid 41 current snap is 337 2024-09-21T10:03:11.719 INFO:tasks.rados.rados.0.smithi033.stdout:3736: read oid 44 snap -1 2024-09-21T10:03:11.719 INFO:tasks.rados.rados.0.smithi033.stdout:3736: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:11.719 INFO:tasks.rados.rados.0.smithi033.stdout:3737: read oid 6 snap -1 2024-09-21T10:03:11.719 INFO:tasks.rados.rados.0.smithi033.stdout:3737: expect (ObjNum 1146 snap 329 seq_num 1146) 2024-09-21T10:03:11.719 INFO:tasks.rados.rados.0.smithi033.stdout:3738: read oid 35 snap -1 2024-09-21T10:03:11.719 INFO:tasks.rados.rados.0.smithi033.stdout:3738: expect deleted 2024-09-21T10:03:11.720 INFO:tasks.rados.rados.0.smithi033.stdout:3739: delete oid 45 current snap is 337 2024-09-21T10:03:11.723 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 41 v 2422 (ObjNum 273 snap 0 seq_num 0) dirty exists 2024-09-21T10:03:11.726 INFO:tasks.rados.rados.0.smithi033.stdout:3734: finishing write tid 1 to smithi03385152-2 2024-09-21T10:03:11.726 INFO:tasks.rados.rados.0.smithi033.stdout:3734: finishing write tid 2 to smithi03385152-2 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:3734: finishing write tid 3 to smithi03385152-2 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 2409 (ObjNum 1176 snap 337 seq_num 1176) dirty exists 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:3734: left oid 2 (ObjNum 1176 snap 337 seq_num 1176) 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:3734: done (5 left) 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:3735: done (4 left) 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:3738: done (3 left) 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:3739: done (2 left) 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:3740: read oid 41 snap -1 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:3740: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:03:11.728 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 337 2024-09-21T10:03:11.729 INFO:tasks.rados.rados.0.smithi033.stdout:3741: seq_num 1178 ranges {319488=532480} 2024-09-21T10:03:11.732 INFO:tasks.rados.rados.0.smithi033.stdout:3741: writing smithi03385152-7 from 319488 to 851968 tid 1 2024-09-21T10:03:11.732 INFO:tasks.rados.rados.0.smithi033.stdout:3742: delete oid 32 current snap is 337 2024-09-21T10:03:11.733 INFO:tasks.rados.rados.0.smithi033.stdout:3740: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:03:11.735 INFO:tasks.rados.rados.0.smithi033.stdout:3740: done (4 left) 2024-09-21T10:03:11.735 INFO:tasks.rados.rados.0.smithi033.stdout:3742: done (3 left) 2024-09-21T10:03:11.735 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 337 2024-09-21T10:03:11.735 INFO:tasks.rados.rados.0.smithi033.stdout:3743: seq_num 1180 ranges {2416640=565248} 2024-09-21T10:03:11.752 INFO:tasks.rados.rados.0.smithi033.stdout:3743: writing smithi03385152-36 from 2416640 to 2981888 tid 1 2024-09-21T10:03:11.752 INFO:tasks.rados.rados.0.smithi033.stdout:3744: snap_create 2024-09-21T10:03:11.752 INFO:tasks.rados.rados.0.smithi033.stdout:3736: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:11.806 INFO:tasks.rados.rados.0.smithi033.stdout:3737: expect (ObjNum 1146 snap 329 seq_num 1146) 2024-09-21T10:03:11.869 INFO:tasks.rados.rados.0.smithi033.stdout:3741: finishing write tid 1 to smithi03385152-7 2024-09-21T10:03:11.869 INFO:tasks.rados.rados.0.smithi033.stdout:3741: finishing write tid 2 to smithi03385152-7 2024-09-21T10:03:11.869 INFO:tasks.rados.rados.0.smithi033.stdout:3741: finishing write tid 3 to smithi03385152-7 2024-09-21T10:03:11.869 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2663 (ObjNum 1178 snap 337 seq_num 1178) dirty exists 2024-09-21T10:03:11.869 INFO:tasks.rados.rados.0.smithi033.stdout:3741: left oid 7 (ObjNum 1178 snap 337 seq_num 1178) 2024-09-21T10:03:11.869 INFO:tasks.rados.rados.0.smithi033.stdout:3743: finishing write tid 1 to smithi03385152-36 2024-09-21T10:03:11.870 INFO:tasks.rados.rados.0.smithi033.stdout:3743: finishing write tid 2 to smithi03385152-36 2024-09-21T10:03:11.870 INFO:tasks.rados.rados.0.smithi033.stdout:3743: finishing write tid 3 to smithi03385152-36 2024-09-21T10:03:11.870 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2666 (ObjNum 1180 snap 337 seq_num 1180) dirty exists 2024-09-21T10:03:11.870 INFO:tasks.rados.rados.0.smithi033.stdout:3743: left oid 36 (ObjNum 1180 snap 337 seq_num 1180) 2024-09-21T10:03:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:11 smithi033 ceph-mon[98546]: pgmap v762: 121 pgs: 121 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 1 op/s 2024-09-21T10:03:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:11 smithi033 ceph-mon[98546]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T10:03:11.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:03:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:03:11] "GET /metrics HTTP/1.1" 200 36441 "" "Prometheus/2.51.0" 2024-09-21T10:03:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:11 smithi033 ceph-mon[96568]: pgmap v762: 121 pgs: 121 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 1 op/s 2024-09-21T10:03:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:11 smithi033 ceph-mon[96568]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T10:03:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:11 smithi105 ceph-mon[86649]: pgmap v762: 121 pgs: 121 active+clean; 235 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 594 KiB/s wr, 1 op/s 2024-09-21T10:03:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:11 smithi105 ceph-mon[86649]: osdmap e752: 8 total, 8 up, 8 in 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:3736: done (4 left) 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:3737: done (3 left) 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:3741: done (2 left) 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:3743: done (1 left) 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:3744: done (0 left) 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:3745: read oid 2 snap 335 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:3745: expect (ObjNum 1052 snap 310 seq_num 1052) 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 12 current snap is 338 2024-09-21T10:03:12.677 INFO:tasks.rados.rados.0.smithi033.stdout:3746: seq_num 1181 ranges {6856704=581632} 2024-09-21T10:03:12.718 INFO:tasks.rados.rados.0.smithi033.stdout:3746: writing smithi03385152-12 from 6856704 to 7438336 tid 1 2024-09-21T10:03:12.718 INFO:tasks.rados.rados.0.smithi033.stdout:3747: read oid 18 snap -1 2024-09-21T10:03:12.719 INFO:tasks.rados.rados.0.smithi033.stdout:3747: expect (ObjNum 1169 snap 335 seq_num 1169) 2024-09-21T10:03:12.719 INFO:tasks.rados.rados.0.smithi033.stdout:3748: read oid 21 snap -1 2024-09-21T10:03:12.719 INFO:tasks.rados.rados.0.smithi033.stdout:3748: expect (ObjNum 1098 snap 323 seq_num 1098) 2024-09-21T10:03:12.719 INFO:tasks.rados.rados.0.smithi033.stdout:3749: snap_create 2024-09-21T10:03:12.719 INFO:tasks.rados.rados.0.smithi033.stdout:3745: expect (ObjNum 1052 snap 310 seq_num 1052) 2024-09-21T10:03:12.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:12 smithi033 ceph-mon[98546]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T10:03:12.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:12 smithi033 ceph-mon[96568]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T10:03:13.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:12 smithi105 ceph-mon[86649]: osdmap e753: 8 total, 8 up, 8 in 2024-09-21T10:03:13.328 INFO:tasks.rados.rados.0.smithi033.stdout:3746: finishing write tid 1 to smithi03385152-12 2024-09-21T10:03:13.328 INFO:tasks.rados.rados.0.smithi033.stdout:3746: finishing write tid 2 to smithi03385152-12 2024-09-21T10:03:13.328 INFO:tasks.rados.rados.0.smithi033.stdout:3746: finishing write tid 3 to smithi03385152-12 2024-09-21T10:03:13.328 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2492 (ObjNum 1181 snap 338 seq_num 1181) dirty exists 2024-09-21T10:03:13.328 INFO:tasks.rados.rados.0.smithi033.stdout:3746: left oid 12 (ObjNum 1181 snap 338 seq_num 1181) 2024-09-21T10:03:13.328 INFO:tasks.rados.rados.0.smithi033.stdout:3747: expect (ObjNum 1169 snap 335 seq_num 1169) 2024-09-21T10:03:13.464 INFO:tasks.rados.rados.0.smithi033.stdout:3748: expect (ObjNum 1098 snap 323 seq_num 1098) 2024-09-21T10:03:13.571 INFO:tasks.rados.rados.0.smithi033.stdout:3745: done (4 left) 2024-09-21T10:03:13.571 INFO:tasks.rados.rados.0.smithi033.stdout:3746: done (3 left) 2024-09-21T10:03:13.571 INFO:tasks.rados.rados.0.smithi033.stdout:3747: done (2 left) 2024-09-21T10:03:13.571 INFO:tasks.rados.rados.0.smithi033.stdout:3748: done (1 left) 2024-09-21T10:03:13.571 INFO:tasks.rados.rados.0.smithi033.stdout:3749: done (0 left) 2024-09-21T10:03:13.572 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 339 2024-09-21T10:03:13.572 INFO:tasks.rados.rados.0.smithi033.stdout:3750: seq_num 1182 ranges {0=606208} 2024-09-21T10:03:13.575 INFO:tasks.rados.rados.0.smithi033.stdout:3750: writing smithi03385152-32 from 0 to 606208 tid 1 2024-09-21T10:03:13.575 INFO:tasks.rados.rados.0.smithi033.stdout:3751: snap_create 2024-09-21T10:03:13.580 INFO:tasks.rados.rados.0.smithi033.stdout:3750: finishing write tid 1 to smithi03385152-32 2024-09-21T10:03:13.580 INFO:tasks.rados.rados.0.smithi033.stdout:3750: finishing write tid 2 to smithi03385152-32 2024-09-21T10:03:13.581 INFO:tasks.rados.rados.0.smithi033.stdout:3750: finishing write tid 3 to smithi03385152-32 2024-09-21T10:03:13.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1604 (ObjNum 1182 snap 339 seq_num 1182) dirty exists 2024-09-21T10:03:13.581 INFO:tasks.rados.rados.0.smithi033.stdout:3750: left oid 32 (ObjNum 1182 snap 339 seq_num 1182) 2024-09-21T10:03:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:13 smithi033 ceph-mon[98546]: pgmap v765: 121 pgs: 121 active+clean; 231 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 606 KiB/s wr, 7 op/s 2024-09-21T10:03:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:13 smithi033 ceph-mon[98546]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T10:03:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:13 smithi033 ceph-mon[98546]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T10:03:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:13 smithi033 ceph-mon[96568]: pgmap v765: 121 pgs: 121 active+clean; 231 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 606 KiB/s wr, 7 op/s 2024-09-21T10:03:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:13 smithi033 ceph-mon[96568]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T10:03:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:13 smithi033 ceph-mon[96568]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T10:03:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:13 smithi105 ceph-mon[86649]: pgmap v765: 121 pgs: 121 active+clean; 231 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 511 B/s rd, 606 KiB/s wr, 7 op/s 2024-09-21T10:03:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:13 smithi105 ceph-mon[86649]: osdmap e754: 8 total, 8 up, 8 in 2024-09-21T10:03:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:13 smithi105 ceph-mon[86649]: osdmap e755: 8 total, 8 up, 8 in 2024-09-21T10:03:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:3750: done (1 left) 2024-09-21T10:03:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:3751: done (0 left) 2024-09-21T10:03:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:3752: read oid 10 snap -1 2024-09-21T10:03:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:3752: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:03:14.577 INFO:tasks.rados.rados.0.smithi033.stdout:3753: snap_remove snap 325 2024-09-21T10:03:15.577 INFO:tasks.rados.rados.0.smithi033.stdout:3753: done (1 left) 2024-09-21T10:03:15.577 INFO:tasks.rados.rados.0.smithi033.stdout:3754: copy_from oid 4 from oid 16 current snap is 340 2024-09-21T10:03:15.577 INFO:tasks.rados.rados.0.smithi033.stdout:3752: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:03:15.634 INFO:tasks.rados.rados.0.smithi033.stdout:3754: finishing copy_from racing read to smithi03385152-4 2024-09-21T10:03:15.634 INFO:tasks.rados.rados.0.smithi033.stdout:3754: finishing copy_from to smithi03385152-4 2024-09-21T10:03:15.634 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 2636 (ObjNum 1163 snap 333 seq_num 1163) dirty exists 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:3752: done (1 left) 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:3754: done (0 left) 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:3755: read oid 20 snap 331 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:3755: expect deleted 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:3756: rollback oid 43 current snap is 340 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 43 to 339 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:3757: rmattr oid 17 current snap is 340 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:3757: done (2 left) 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 340 2024-09-21T10:03:15.635 INFO:tasks.rados.rados.0.smithi033.stdout:3758: seq_num 1183 ranges {147456=622592} 2024-09-21T10:03:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:3758: writing smithi03385152-38 from 147456 to 770048 tid 1 2024-09-21T10:03:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:3759: snap_create 2024-09-21T10:03:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:3756: finishing rollback tid 1 to smithi03385152-43 2024-09-21T10:03:15.638 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 43 v 0 (ObjNum 161 snap 0 seq_num 1) dirty dne 2024-09-21T10:03:15.642 INFO:tasks.rados.rados.0.smithi033.stdout:3758: finishing write tid 1 to smithi03385152-38 2024-09-21T10:03:15.642 INFO:tasks.rados.rados.0.smithi033.stdout:3758: finishing write tid 2 to smithi03385152-38 2024-09-21T10:03:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:3758: finishing write tid 3 to smithi03385152-38 2024-09-21T10:03:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2424 (ObjNum 1183 snap 340 seq_num 1183) dirty exists 2024-09-21T10:03:15.643 INFO:tasks.rados.rados.0.smithi033.stdout:3758: left oid 38 (ObjNum 1183 snap 340 seq_num 1183) 2024-09-21T10:03:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:15 smithi033 ceph-mon[98546]: pgmap v768: 121 pgs: 121 active+clean; 235 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-21T10:03:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:15 smithi033 ceph-mon[98546]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T10:03:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:15 smithi033 ceph-mon[96568]: pgmap v768: 121 pgs: 121 active+clean; 235 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-21T10:03:15.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:15 smithi033 ceph-mon[96568]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T10:03:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:15 smithi105 ceph-mon[86649]: pgmap v768: 121 pgs: 121 active+clean; 235 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.1 MiB/s wr, 20 op/s 2024-09-21T10:03:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:15 smithi105 ceph-mon[86649]: osdmap e756: 8 total, 8 up, 8 in 2024-09-21T10:03:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:3755: done (3 left) 2024-09-21T10:03:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:3756: done (2 left) 2024-09-21T10:03:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:3758: done (1 left) 2024-09-21T10:03:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:3759: done (0 left) 2024-09-21T10:03:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:3760: copy_from oid 19 from oid 27 current snap is 341 2024-09-21T10:03:16.588 INFO:tasks.rados.rados.0.smithi033.stdout:3761: snap_remove snap 335 2024-09-21T10:03:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:16 smithi033 ceph-mon[98546]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T10:03:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:16 smithi033 ceph-mon[96568]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T10:03:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:16 smithi105 ceph-mon[86649]: osdmap e757: 8 total, 8 up, 8 in 2024-09-21T10:03:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:17.590 INFO:tasks.rados.rados.0.smithi033.stdout:3761: done (1 left) 2024-09-21T10:03:17.590 INFO:tasks.rados.rados.0.smithi033.stdout:3762: read oid 21 snap 336 2024-09-21T10:03:17.591 INFO:tasks.rados.rados.0.smithi033.stdout:3762: expect (ObjNum 1098 snap 323 seq_num 1098) 2024-09-21T10:03:17.591 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 7 current snap is 341 2024-09-21T10:03:17.591 INFO:tasks.rados.rados.0.smithi033.stdout:3763: seq_num 1184 ranges {851968=638976} 2024-09-21T10:03:17.596 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:17 smithi033 ceph-mon[96568]: pgmap v771: 121 pgs: 121 active+clean; 201 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 19 op/s 2024-09-21T10:03:17.599 INFO:tasks.rados.rados.0.smithi033.stdout:3763: writing smithi03385152-7 from 851968 to 1490944 tid 1 2024-09-21T10:03:17.600 INFO:tasks.rados.rados.0.smithi033.stdout:3764: read oid 38 snap 337 2024-09-21T10:03:17.600 INFO:tasks.rados.rados.0.smithi033.stdout:3764: expect (ObjNum 1155 snap 330 seq_num 1155) 2024-09-21T10:03:17.600 INFO:tasks.rados.rados.0.smithi033.stdout:3765: delete oid 1 current snap is 341 2024-09-21T10:03:17.600 INFO:tasks.rados.rados.0.smithi033.stdout:3760: finishing copy_from racing read to smithi03385152-19 2024-09-21T10:03:17.600 INFO:tasks.rados.rados.0.smithi033.stdout:3760: finishing copy_from to smithi03385152-19 2024-09-21T10:03:17.600 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2485 (ObjNum 1127 snap 325 seq_num 1127) dirty exists 2024-09-21T10:03:17.605 INFO:tasks.rados.rados.0.smithi033.stdout:3760: done (4 left) 2024-09-21T10:03:17.605 INFO:tasks.rados.rados.0.smithi033.stdout:3765: done (3 left) 2024-09-21T10:03:17.605 INFO:tasks.rados.rados.0.smithi033.stdout:3766: read oid 37 snap -1 2024-09-21T10:03:17.605 INFO:tasks.rados.rados.0.smithi033.stdout:3766: expect deleted 2024-09-21T10:03:17.605 INFO:tasks.rados.rados.0.smithi033.stdout:3767: snap_remove snap 338 2024-09-21T10:03:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:17 smithi033 ceph-mon[96568]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T10:03:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:17 smithi033 ceph-mon[98546]: pgmap v771: 121 pgs: 121 active+clean; 201 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 19 op/s 2024-09-21T10:03:17.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:17 smithi033 ceph-mon[98546]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T10:03:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:17 smithi105 ceph-mon[86649]: pgmap v771: 121 pgs: 121 active+clean; 201 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 17 MiB/s rd, 1.4 MiB/s wr, 19 op/s 2024-09-21T10:03:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:17 smithi105 ceph-mon[86649]: osdmap e758: 8 total, 8 up, 8 in 2024-09-21T10:03:18.576 INFO:tasks.rados.rados.0.smithi033.stdout:3767: done (4 left) 2024-09-21T10:03:18.576 INFO:tasks.rados.rados.0.smithi033.stdout:3768: snap_remove snap 339 2024-09-21T10:03:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:18 smithi033 ceph-mon[98546]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T10:03:18.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:18 smithi033 ceph-mon[98546]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T10:03:18.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:18 smithi033 ceph-mon[96568]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T10:03:18.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:18 smithi033 ceph-mon[96568]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T10:03:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:18 smithi105 ceph-mon[86649]: osdmap e759: 8 total, 8 up, 8 in 2024-09-21T10:03:19.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:18 smithi105 ceph-mon[86649]: osdmap e760: 8 total, 8 up, 8 in 2024-09-21T10:03:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:3768: done (4 left) 2024-09-21T10:03:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:3769: snap_create 2024-09-21T10:03:19.579 INFO:tasks.rados.rados.0.smithi033.stdout:3762: expect (ObjNum 1098 snap 323 seq_num 1098) 2024-09-21T10:03:19.685 INFO:tasks.rados.rados.0.smithi033.stdout:3763: finishing write tid 1 to smithi03385152-7 2024-09-21T10:03:19.685 INFO:tasks.rados.rados.0.smithi033.stdout:3763: finishing write tid 2 to smithi03385152-7 2024-09-21T10:03:19.685 INFO:tasks.rados.rados.0.smithi033.stdout:3764: expect (ObjNum 1155 snap 330 seq_num 1155) 2024-09-21T10:03:19.688 INFO:tasks.rados.rados.0.smithi033.stdout:3763: finishing write tid 3 to smithi03385152-7 2024-09-21T10:03:19.689 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 2681 (ObjNum 1184 snap 341 seq_num 1184) dirty exists 2024-09-21T10:03:19.689 INFO:tasks.rados.rados.0.smithi033.stdout:3763: left oid 7 (ObjNum 1184 snap 341 seq_num 1184) 2024-09-21T10:03:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:19 smithi033 ceph-mon[98546]: pgmap v774: 121 pgs: 121 active+clean; 195 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 390 KiB/s wr, 5 op/s 2024-09-21T10:03:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:19 smithi033 ceph-mon[98546]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T10:03:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:19 smithi033 ceph-mon[96568]: pgmap v774: 121 pgs: 121 active+clean; 195 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 390 KiB/s wr, 5 op/s 2024-09-21T10:03:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:19 smithi033 ceph-mon[96568]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T10:03:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:19 smithi105 ceph-mon[86649]: pgmap v774: 121 pgs: 121 active+clean; 195 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 9.6 MiB/s rd, 390 KiB/s wr, 5 op/s 2024-09-21T10:03:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:19 smithi105 ceph-mon[86649]: osdmap e761: 8 total, 8 up, 8 in 2024-09-21T10:03:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:03:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:03:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:20.579 INFO:tasks.rados.rados.0.smithi033.stdout:3762: done (4 left) 2024-09-21T10:03:20.579 INFO:tasks.rados.rados.0.smithi033.stdout:3763: done (3 left) 2024-09-21T10:03:20.579 INFO:tasks.rados.rados.0.smithi033.stdout:3764: done (2 left) 2024-09-21T10:03:20.579 INFO:tasks.rados.rados.0.smithi033.stdout:3766: done (1 left) 2024-09-21T10:03:20.579 INFO:tasks.rados.rados.0.smithi033.stdout:3769: done (0 left) 2024-09-21T10:03:20.579 INFO:tasks.rados.rados.0.smithi033.stdout:3770: delete oid 7 current snap is 342 2024-09-21T10:03:20.585 INFO:tasks.rados.rados.0.smithi033.stdout:3770: done (0 left) 2024-09-21T10:03:20.585 INFO:tasks.rados.rados.0.smithi033.stdout:3771: setattr oid 17 current snap is 342 2024-09-21T10:03:20.586 INFO:tasks.rados.rados.0.smithi033.stdout:3772: snap_remove snap 336 2024-09-21T10:03:21.580 INFO:tasks.rados.rados.0.smithi033.stdout:3772: done (1 left) 2024-09-21T10:03:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:3773: snap_create 2024-09-21T10:03:21.581 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2493 (ObjNum 1103 snap 324 seq_num 1103) dirty exists 2024-09-21T10:03:21.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:03:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:03:21] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.51.0" 2024-09-21T10:03:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:21 smithi033 ceph-mon[96568]: pgmap v777: 121 pgs: 121 active+clean; 173 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T10:03:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:21 smithi033 ceph-mon[96568]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T10:03:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:21 smithi033 ceph-mon[98546]: pgmap v777: 121 pgs: 121 active+clean; 173 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T10:03:21.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:21 smithi033 ceph-mon[98546]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T10:03:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:21 smithi105 ceph-mon[86649]: pgmap v777: 121 pgs: 121 active+clean; 173 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-09-21T10:03:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:21 smithi105 ceph-mon[86649]: osdmap e762: 8 total, 8 up, 8 in 2024-09-21T10:03:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3771: done (1 left) 2024-09-21T10:03:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3773: done (0 left) 2024-09-21T10:03:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 343 2024-09-21T10:03:22.581 INFO:tasks.rados.rados.0.smithi033.stdout:3774: seq_num 1187 ranges {2170880=688128} 2024-09-21T10:03:22.597 INFO:tasks.rados.rados.0.smithi033.stdout:3774: writing smithi03385152-46 from 2170880 to 2859008 tid 1 2024-09-21T10:03:22.597 INFO:tasks.rados.rados.0.smithi033.stdout:3775: read oid 11 snap -1 2024-09-21T10:03:22.597 INFO:tasks.rados.rados.0.smithi033.stdout:3775: expect (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:03:22.597 INFO:tasks.rados.rados.0.smithi033.stdout:3776: setattr oid 38 current snap is 343 2024-09-21T10:03:22.598 INFO:tasks.rados.rados.0.smithi033.stdout:3777: snap_remove snap 333 2024-09-21T10:03:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:22 smithi033 ceph-mon[98546]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T10:03:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:22 smithi033 ceph-mon[98546]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T10:03:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:22 smithi033 ceph-mon[96568]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T10:03:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:22 smithi033 ceph-mon[96568]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T10:03:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:22 smithi105 ceph-mon[86649]: osdmap e763: 8 total, 8 up, 8 in 2024-09-21T10:03:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:22 smithi105 ceph-mon[86649]: osdmap e764: 8 total, 8 up, 8 in 2024-09-21T10:03:23.579 INFO:tasks.rados.rados.0.smithi033.stdout:3777: done (3 left) 2024-09-21T10:03:23.579 INFO:tasks.rados.rados.0.smithi033.stdout:3778: snap_remove snap 340 2024-09-21T10:03:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:23 smithi033 ceph-mon[98546]: pgmap v780: 121 pgs: 121 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-21T10:03:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:23 smithi033 ceph-mon[98546]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T10:03:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:23 smithi033 ceph-mon[96568]: pgmap v780: 121 pgs: 121 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-21T10:03:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:23 smithi033 ceph-mon[96568]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T10:03:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:23 smithi105 ceph-mon[86649]: pgmap v780: 121 pgs: 121 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-21T10:03:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:23 smithi105 ceph-mon[86649]: osdmap e765: 8 total, 8 up, 8 in 2024-09-21T10:03:24.582 INFO:tasks.rados.rados.0.smithi033.stdout:3778: done (3 left) 2024-09-21T10:03:24.582 INFO:tasks.rados.rados.0.smithi033.stdout:3779: snap_create 2024-09-21T10:03:24.582 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2425 (ObjNum 1183 snap 340 seq_num 1183) dirty exists 2024-09-21T10:03:24.582 INFO:tasks.rados.rados.0.smithi033.stdout:3774: finishing write tid 1 to smithi03385152-46 2024-09-21T10:03:24.582 INFO:tasks.rados.rados.0.smithi033.stdout:3774: finishing write tid 2 to smithi03385152-46 2024-09-21T10:03:24.582 INFO:tasks.rados.rados.0.smithi033.stdout:3774: finishing write tid 3 to smithi03385152-46 2024-09-21T10:03:24.582 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2486 (ObjNum 1187 snap 343 seq_num 1187) dirty exists 2024-09-21T10:03:24.582 INFO:tasks.rados.rados.0.smithi033.stdout:3774: left oid 46 (ObjNum 1187 snap 343 seq_num 1187) 2024-09-21T10:03:24.583 INFO:tasks.rados.rados.0.smithi033.stdout:3775: expect (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:03:25.579 INFO:tasks.rados.rados.0.smithi033.stdout:3774: done (3 left) 2024-09-21T10:03:25.579 INFO:tasks.rados.rados.0.smithi033.stdout:3775: done (2 left) 2024-09-21T10:03:25.579 INFO:tasks.rados.rados.0.smithi033.stdout:3776: done (1 left) 2024-09-21T10:03:25.579 INFO:tasks.rados.rados.0.smithi033.stdout:3779: done (0 left) 2024-09-21T10:03:25.579 INFO:tasks.rados.rados.0.smithi033.stdout:3780: rmattr oid 20 current snap is 344 2024-09-21T10:03:25.579 INFO:tasks.rados.rados.0.smithi033.stdout:3780: done (0 left) 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:3781: rollback oid 4 current snap is 344 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 4 to 343 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:3782: rmattr oid 11 current snap is 344 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:3782: done (1 left) 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:3783: read oid 23 snap 331 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:3783: expect (ObjNum 1083 snap 319 seq_num 1083) 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:3784: read oid 22 snap 343 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:3784: expect (ObjNum 1134 snap 327 seq_num 1134) 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 344 2024-09-21T10:03:25.580 INFO:tasks.rados.rados.0.smithi033.stdout:3785: seq_num 1188 ranges {2981888=704512} 2024-09-21T10:03:25.600 INFO:tasks.rados.rados.0.smithi033.stdout:3785: writing smithi03385152-36 from 2981888 to 3686400 tid 1 2024-09-21T10:03:25.600 INFO:tasks.rados.rados.0.smithi033.stdout:3786: setattr oid 14 current snap is 344 2024-09-21T10:03:25.600 INFO:tasks.rados.rados.0.smithi033.stdout:3781: finishing rollback tid 0 to smithi03385152-4 2024-09-21T10:03:25.600 INFO:tasks.rados.rados.0.smithi033.stdout:3783: expect (ObjNum 1083 snap 319 seq_num 1083) 2024-09-21T10:03:25.689 INFO:tasks.rados.rados.0.smithi033.stdout:3781: finishing rollback tid 1 to smithi03385152-4 2024-09-21T10:03:25.689 INFO:tasks.rados.rados.0.smithi033.stdout:3781: finishing rollback tid 2 to smithi03385152-4 2024-09-21T10:03:25.689 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 4 v 2639 (ObjNum 1163 snap 333 seq_num 1163) dirty exists 2024-09-21T10:03:25.689 INFO:tasks.rados.rados.0.smithi033.stdout:3784: expect (ObjNum 1134 snap 327 seq_num 1134) 2024-09-21T10:03:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:25 smithi033 ceph-mon[98546]: pgmap v783: 121 pgs: 121 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 156 KiB/s wr, 12 op/s 2024-09-21T10:03:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:25 smithi033 ceph-mon[98546]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T10:03:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:25 smithi033 ceph-mon[96568]: pgmap v783: 121 pgs: 121 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 156 KiB/s wr, 12 op/s 2024-09-21T10:03:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:25 smithi033 ceph-mon[96568]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T10:03:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:25 smithi105 ceph-mon[86649]: pgmap v783: 121 pgs: 121 active+clean; 174 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 156 KiB/s wr, 12 op/s 2024-09-21T10:03:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:25 smithi105 ceph-mon[86649]: osdmap e766: 8 total, 8 up, 8 in 2024-09-21T10:03:26.174 INFO:tasks.rados.rados.0.smithi033.stdout:3785: finishing write tid 1 to smithi03385152-36 2024-09-21T10:03:26.174 INFO:tasks.rados.rados.0.smithi033.stdout:3785: finishing write tid 2 to smithi03385152-36 2024-09-21T10:03:26.174 INFO:tasks.rados.rados.0.smithi033.stdout:3785: finishing write tid 3 to smithi03385152-36 2024-09-21T10:03:26.174 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2700 (ObjNum 1188 snap 344 seq_num 1188) dirty exists 2024-09-21T10:03:26.174 INFO:tasks.rados.rados.0.smithi033.stdout:3785: left oid 36 (ObjNum 1188 snap 344 seq_num 1188) 2024-09-21T10:03:26.174 INFO:tasks.rados.rados.0.smithi033.stdout:3781: done (4 left) 2024-09-21T10:03:26.174 INFO:tasks.rados.rados.0.smithi033.stdout:3783: done (3 left) 2024-09-21T10:03:26.175 INFO:tasks.rados.rados.0.smithi033.stdout:3784: done (2 left) 2024-09-21T10:03:26.175 INFO:tasks.rados.rados.0.smithi033.stdout:3785: done (1 left) 2024-09-21T10:03:26.175 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 344 2024-09-21T10:03:26.175 INFO:tasks.rados.rados.0.smithi033.stdout:3787: seq_num 1189 ranges {2506752=720896} 2024-09-21T10:03:26.191 INFO:tasks.rados.rados.0.smithi033.stdout:3787: writing smithi03385152-9 from 2506752 to 3227648 tid 1 2024-09-21T10:03:26.191 INFO:tasks.rados.rados.0.smithi033.stdout:3788: snap_remove snap 341 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2487 (ObjNum 1001 snap 292 seq_num 1001) dirty exists 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:3787: finishing write tid 1 to smithi03385152-9 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:3787: finishing write tid 2 to smithi03385152-9 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:3787: finishing write tid 3 to smithi03385152-9 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 2427 (ObjNum 1189 snap 344 seq_num 1189) dirty exists 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:3787: left oid 9 (ObjNum 1189 snap 344 seq_num 1189) 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:3786: done (2 left) 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:3787: done (1 left) 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:3788: done (0 left) 2024-09-21T10:03:26.596 INFO:tasks.rados.rados.0.smithi033.stdout:3789: setattr oid 27 current snap is 344 2024-09-21T10:03:26.597 INFO:tasks.rados.rados.0.smithi033.stdout:3790: read oid 44 snap -1 2024-09-21T10:03:26.597 INFO:tasks.rados.rados.0.smithi033.stdout:3790: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:26.597 INFO:tasks.rados.rados.0.smithi033.stdout:3791: read oid 50 snap -1 2024-09-21T10:03:26.597 INFO:tasks.rados.rados.0.smithi033.stdout:3791: expect (ObjNum 1152 snap 330 seq_num 1152) 2024-09-21T10:03:26.598 INFO:tasks.rados.rados.0.smithi033.stdout:3792: copy_from oid 42 from oid 11 current snap is 344 2024-09-21T10:03:26.598 INFO:tasks.rados.rados.0.smithi033.stdout:3793: rmattr oid 19 current snap is 344 2024-09-21T10:03:26.598 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 8 current snap is 344 2024-09-21T10:03:26.598 INFO:tasks.rados.rados.0.smithi033.stdout:3794: seq_num 1190 ranges {2039808=737280} 2024-09-21T10:03:26.614 INFO:tasks.rados.rados.0.smithi033.stdout:3794: writing smithi03385152-8 from 2039808 to 2777088 tid 1 2024-09-21T10:03:26.614 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 344 2024-09-21T10:03:26.614 INFO:tasks.rados.rados.0.smithi033.stdout:3795: seq_num 1191 ranges {6299648=753664} 2024-09-21T10:03:26.654 INFO:tasks.rados.rados.0.smithi033.stdout:3795: writing smithi03385152-29 from 6299648 to 7053312 tid 1 2024-09-21T10:03:26.655 INFO:tasks.rados.rados.0.smithi033.stdout:3796: snap_remove snap 331 2024-09-21T10:03:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:26 smithi033 ceph-mon[96568]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T10:03:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:26 smithi033 ceph-mon[98546]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T10:03:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:26 smithi105 ceph-mon[86649]: osdmap e767: 8 total, 8 up, 8 in 2024-09-21T10:03:27.602 INFO:tasks.rados.rados.0.smithi033.stdout:3796: done (7 left) 2024-09-21T10:03:27.602 INFO:tasks.rados.rados.0.smithi033.stdout:3797: snap_create 2024-09-21T10:03:27.602 INFO:tasks.rados.rados.0.smithi033.stdout:3791: expect (ObjNum 1152 snap 330 seq_num 1152) 2024-09-21T10:03:27.607 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2489 (ObjNum 1127 snap 325 seq_num 1127) dirty exists 2024-09-21T10:03:27.607 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2498 (ObjNum 1127 snap 325 seq_num 1127) dirty exists 2024-09-21T10:03:27.607 INFO:tasks.rados.rados.0.smithi033.stdout:3790: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3794: finishing write tid 1 to smithi03385152-8 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3794: finishing write tid 2 to smithi03385152-8 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3794: finishing write tid 3 to smithi03385152-8 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 8 v 2475 (ObjNum 1190 snap 344 seq_num 1190) dirty exists 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3794: left oid 8 (ObjNum 1190 snap 344 seq_num 1190) 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3792: finishing copy_from racing read to smithi03385152-42 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3792: finishing copy_from to smithi03385152-42 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 2487 (ObjNum 1136 snap 327 seq_num 1136) dirty exists 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3795: finishing write tid 1 to smithi03385152-29 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3795: finishing write tid 2 to smithi03385152-29 2024-09-21T10:03:27.667 INFO:tasks.rados.rados.0.smithi033.stdout:3795: finishing write tid 3 to smithi03385152-29 2024-09-21T10:03:27.668 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2707 (ObjNum 1191 snap 344 seq_num 1191) dirty exists 2024-09-21T10:03:27.668 INFO:tasks.rados.rados.0.smithi033.stdout:3795: left oid 29 (ObjNum 1191 snap 344 seq_num 1191) 2024-09-21T10:03:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:27 smithi033 ceph-mon[96568]: pgmap v786: 121 pgs: 121 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 340 KiB/s wr, 14 op/s 2024-09-21T10:03:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:27 smithi033 ceph-mon[96568]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T10:03:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:27 smithi033 ceph-mon[98546]: pgmap v786: 121 pgs: 121 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 340 KiB/s wr, 14 op/s 2024-09-21T10:03:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:27 smithi033 ceph-mon[98546]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T10:03:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:27 smithi105 ceph-mon[86649]: pgmap v786: 121 pgs: 121 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 340 KiB/s wr, 14 op/s 2024-09-21T10:03:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:27 smithi105 ceph-mon[86649]: osdmap e768: 8 total, 8 up, 8 in 2024-09-21T10:03:28.584 INFO:tasks.rados.rados.0.smithi033.stdout:3789: done (7 left) 2024-09-21T10:03:28.584 INFO:tasks.rados.rados.0.smithi033.stdout:3790: done (6 left) 2024-09-21T10:03:28.584 INFO:tasks.rados.rados.0.smithi033.stdout:3791: done (5 left) 2024-09-21T10:03:28.584 INFO:tasks.rados.rados.0.smithi033.stdout:3792: done (4 left) 2024-09-21T10:03:28.584 INFO:tasks.rados.rados.0.smithi033.stdout:3793: done (3 left) 2024-09-21T10:03:28.585 INFO:tasks.rados.rados.0.smithi033.stdout:3794: done (2 left) 2024-09-21T10:03:28.585 INFO:tasks.rados.rados.0.smithi033.stdout:3795: done (1 left) 2024-09-21T10:03:28.585 INFO:tasks.rados.rados.0.smithi033.stdout:3797: done (0 left) 2024-09-21T10:03:28.585 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 17 current snap is 345 2024-09-21T10:03:28.585 INFO:tasks.rados.rados.0.smithi033.stdout:3798: seq_num 1192 ranges {1679360=770048} 2024-09-21T10:03:28.597 INFO:tasks.rados.rados.0.smithi033.stdout:3798: writing smithi03385152-17 from 1679360 to 2449408 tid 1 2024-09-21T10:03:28.597 INFO:tasks.rados.rados.0.smithi033.stdout:3799: read oid 7 snap -1 2024-09-21T10:03:28.597 INFO:tasks.rados.rados.0.smithi033.stdout:3799: expect deleted 2024-09-21T10:03:28.598 INFO:tasks.rados.rados.0.smithi033.stdout:3800: snap_remove snap 342 2024-09-21T10:03:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:28 smithi033 ceph-mon[98546]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T10:03:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:28 smithi033 ceph-mon[98546]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T10:03:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:28 smithi033 ceph-mon[96568]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T10:03:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:28 smithi033 ceph-mon[96568]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T10:03:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:28 smithi105 ceph-mon[86649]: osdmap e769: 8 total, 8 up, 8 in 2024-09-21T10:03:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:28 smithi105 ceph-mon[86649]: osdmap e770: 8 total, 8 up, 8 in 2024-09-21T10:03:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:3800: done (2 left) 2024-09-21T10:03:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 345 2024-09-21T10:03:29.581 INFO:tasks.rados.rados.0.smithi033.stdout:3801: seq_num 1193 ranges {286720=786432} 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3801: writing smithi03385152-16 from 286720 to 1073152 tid 1 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3802: rmattr oid 29 current snap is 345 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3799: done (3 left) 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 345 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3798: finishing write tid 1 to smithi03385152-17 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3798: finishing write tid 2 to smithi03385152-17 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3798: finishing write tid 3 to smithi03385152-17 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2495 (ObjNum 1192 snap 345 seq_num 1192) dirty exists 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3798: left oid 17 (ObjNum 1192 snap 345 seq_num 1192) 2024-09-21T10:03:29.589 INFO:tasks.rados.rados.0.smithi033.stdout:3803: seq_num 1194 ranges {1679360=802816} 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:3803: writing smithi03385152-10 from 1679360 to 2482176 tid 1 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:3798: done (3 left) 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:3804: delete oid 36 current snap is 345 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:3801: finishing write tid 1 to smithi03385152-16 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:3801: finishing write tid 2 to smithi03385152-16 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2724 (ObjNum 1191 snap 344 seq_num 1191) dirty exists 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:3801: finishing write tid 3 to smithi03385152-16 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2722 (ObjNum 1193 snap 345 seq_num 1193) dirty exists 2024-09-21T10:03:29.603 INFO:tasks.rados.rados.0.smithi033.stdout:3801: left oid 16 (ObjNum 1193 snap 345 seq_num 1193) 2024-09-21T10:03:29.606 INFO:tasks.rados.rados.0.smithi033.stdout:3801: done (3 left) 2024-09-21T10:03:29.606 INFO:tasks.rados.rados.0.smithi033.stdout:3802: done (2 left) 2024-09-21T10:03:29.606 INFO:tasks.rados.rados.0.smithi033.stdout:3804: done (1 left) 2024-09-21T10:03:29.606 INFO:tasks.rados.rados.0.smithi033.stdout:3805: copy_from oid 35 from oid 7 current snap is 345 2024-09-21T10:03:29.606 INFO:tasks.rados.rados.0.smithi033.stdout:3806: rmattr oid 43 current snap is 345 2024-09-21T10:03:29.607 INFO:tasks.rados.rados.0.smithi033.stdout:3806: done (2 left) 2024-09-21T10:03:29.607 INFO:tasks.rados.rados.0.smithi033.stdout:3807: snap_remove snap 344 2024-09-21T10:03:29.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:29 smithi105 ceph-mon[86649]: pgmap v789: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 340 KiB/s wr, 4 op/s 2024-09-21T10:03:29.932 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:29 smithi105 ceph-mon[86649]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T10:03:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:29 smithi033 ceph-mon[98546]: pgmap v789: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 340 KiB/s wr, 4 op/s 2024-09-21T10:03:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:29 smithi033 ceph-mon[98546]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T10:03:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:29 smithi033 ceph-mon[96568]: pgmap v789: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 175 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 340 KiB/s wr, 4 op/s 2024-09-21T10:03:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:29 smithi033 ceph-mon[96568]: osdmap e771: 8 total, 8 up, 8 in 2024-09-21T10:03:30.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:03:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:03:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3807: done (2 left) 2024-09-21T10:03:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3808: setattr oid 27 current snap is 345 2024-09-21T10:03:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3803: finishing write tid 1 to smithi03385152-10 2024-09-21T10:03:30.584 INFO:tasks.rados.rados.0.smithi033.stdout:3803: finishing write tid 2 to smithi03385152-10 2024-09-21T10:03:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:3805: finishing copy_from to smithi03385152-35 2024-09-21T10:03:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:3805: got expected ENOENT (src dne) 2024-09-21T10:03:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:3805: finishing copy_from racing read to smithi03385152-35 2024-09-21T10:03:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:3803: finishing write tid 3 to smithi03385152-10 2024-09-21T10:03:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 2491 (ObjNum 1194 snap 345 seq_num 1194) dirty exists 2024-09-21T10:03:30.585 INFO:tasks.rados.rados.0.smithi033.stdout:3803: left oid 10 (ObjNum 1194 snap 345 seq_num 1194) 2024-09-21T10:03:30.586 INFO:tasks.rados.rados.0.smithi033.stdout:3803: done (2 left) 2024-09-21T10:03:30.586 INFO:tasks.rados.rados.0.smithi033.stdout:3805: done (1 left) 2024-09-21T10:03:30.586 INFO:tasks.rados.rados.0.smithi033.stdout:3809: copy_from oid 30 from oid 9 current snap is 345 2024-09-21T10:03:30.586 INFO:tasks.rados.rados.0.smithi033.stdout:3810: read oid 44 snap -1 2024-09-21T10:03:30.586 INFO:tasks.rados.rados.0.smithi033.stdout:3810: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:30.587 INFO:tasks.rados.rados.0.smithi033.stdout:3811: read oid 49 snap 343 2024-09-21T10:03:30.587 INFO:tasks.rados.rados.0.smithi033.stdout:3811: expect (ObjNum 1170 snap 335 seq_num 1170) 2024-09-21T10:03:30.587 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 3 current snap is 345 2024-09-21T10:03:30.587 INFO:tasks.rados.rados.0.smithi033.stdout:3812: seq_num 1196 ranges {114688=434176,548864=401408} 2024-09-21T10:03:30.589 INFO:tasks.rados.rados.0.smithi033.stdout:3812: writing smithi03385152-3 from 114688 to 548864 tid 1 2024-09-21T10:03:30.591 INFO:tasks.rados.rados.0.smithi033.stdout:3812: writing smithi03385152-3 from 548864 to 950272 tid 2 2024-09-21T10:03:30.591 INFO:tasks.rados.rados.0.smithi033.stdout:3813: snap_remove snap 337 2024-09-21T10:03:31.587 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2499 (ObjNum 1127 snap 325 seq_num 1127) dirty exists 2024-09-21T10:03:31.587 INFO:tasks.rados.rados.0.smithi033.stdout:3811: expect (ObjNum 1170 snap 335 seq_num 1170) 2024-09-21T10:03:31.603 INFO:tasks.rados.rados.0.smithi033.stdout:3812: finishing write tid 1 to smithi03385152-3 2024-09-21T10:03:31.603 INFO:tasks.rados.rados.0.smithi033.stdout:3808: done (5 left) 2024-09-21T10:03:31.603 INFO:tasks.rados.rados.0.smithi033.stdout:3811: done (4 left) 2024-09-21T10:03:31.603 INFO:tasks.rados.rados.0.smithi033.stdout:3813: done (3 left) 2024-09-21T10:03:31.603 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 11 current snap is 345 2024-09-21T10:03:31.603 INFO:tasks.rados.rados.0.smithi033.stdout:3814: seq_num 1197 ranges {9641984=450560,10092544=401408} 2024-09-21T10:03:31.658 INFO:tasks.rados.rados.0.smithi033.stdout:3814: writing smithi03385152-11 from 9641984 to 10092544 tid 1 2024-09-21T10:03:31.660 INFO:tasks.rados.rados.0.smithi033.stdout:3814: writing smithi03385152-11 from 10092544 to 10493952 tid 2 2024-09-21T10:03:31.660 INFO:tasks.rados.rados.0.smithi033.stdout:3815: snap_create 2024-09-21T10:03:31.661 INFO:tasks.rados.rados.0.smithi033.stdout:3812: finishing write tid 2 to smithi03385152-3 2024-09-21T10:03:31.661 INFO:tasks.rados.rados.0.smithi033.stdout:3812: finishing write tid 3 to smithi03385152-3 2024-09-21T10:03:31.661 INFO:tasks.rados.rados.0.smithi033.stdout:3810: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:31.713 INFO:tasks.rados.rados.0.smithi033.stdout:3809: finishing copy_from racing read to smithi03385152-30 2024-09-21T10:03:31.713 INFO:tasks.rados.rados.0.smithi033.stdout:3812: finishing write tid 4 to smithi03385152-3 2024-09-21T10:03:31.713 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2490 (ObjNum 1196 snap 345 seq_num 1196) dirty exists 2024-09-21T10:03:31.713 INFO:tasks.rados.rados.0.smithi033.stdout:3812: left oid 3 (ObjNum 1196 snap 345 seq_num 1196) 2024-09-21T10:03:31.713 INFO:tasks.rados.rados.0.smithi033.stdout:3809: finishing copy_from to smithi03385152-30 2024-09-21T10:03:31.714 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2427 (ObjNum 1189 snap 344 seq_num 1189) dirty exists 2024-09-21T10:03:31.714 INFO:tasks.rados.rados.0.smithi033.stdout:3814: finishing write tid 1 to smithi03385152-11 2024-09-21T10:03:31.714 INFO:tasks.rados.rados.0.smithi033.stdout:3814: finishing write tid 2 to smithi03385152-11 2024-09-21T10:03:31.714 INFO:tasks.rados.rados.0.smithi033.stdout:3814: finishing write tid 3 to smithi03385152-11 2024-09-21T10:03:31.714 INFO:tasks.rados.rados.0.smithi033.stdout:3814: finishing write tid 4 to smithi03385152-11 2024-09-21T10:03:31.714 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 2491 (ObjNum 1197 snap 345 seq_num 1197) dirty exists 2024-09-21T10:03:31.714 INFO:tasks.rados.rados.0.smithi033.stdout:3814: left oid 11 (ObjNum 1197 snap 345 seq_num 1197) 2024-09-21T10:03:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:31 smithi033 ceph-mon[96568]: pgmap v792: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-21T10:03:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:31 smithi033 ceph-mon[96568]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T10:03:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:31 smithi033 ceph-mon[98546]: pgmap v792: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-21T10:03:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:31 smithi033 ceph-mon[98546]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T10:03:31.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:31.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:03:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:03:31] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-09-21T10:03:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:31 smithi105 ceph-mon[86649]: pgmap v792: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 177 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.7 MiB/s wr, 10 op/s 2024-09-21T10:03:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:31 smithi105 ceph-mon[86649]: osdmap e772: 8 total, 8 up, 8 in 2024-09-21T10:03:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:32.604 INFO:tasks.rados.rados.0.smithi033.stdout:3809: done (4 left) 2024-09-21T10:03:32.604 INFO:tasks.rados.rados.0.smithi033.stdout:3810: done (3 left) 2024-09-21T10:03:32.605 INFO:tasks.rados.rados.0.smithi033.stdout:3812: done (2 left) 2024-09-21T10:03:32.605 INFO:tasks.rados.rados.0.smithi033.stdout:3814: done (1 left) 2024-09-21T10:03:32.605 INFO:tasks.rados.rados.0.smithi033.stdout:3815: done (0 left) 2024-09-21T10:03:32.605 INFO:tasks.rados.rados.0.smithi033.stdout:3816: read oid 3 snap -1 2024-09-21T10:03:32.605 INFO:tasks.rados.rados.0.smithi033.stdout:3816: expect (ObjNum 1196 snap 345 seq_num 1196) 2024-09-21T10:03:32.605 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 346 2024-09-21T10:03:32.605 INFO:tasks.rados.rados.0.smithi033.stdout:3817: seq_num 1198 ranges {8421376=466944,8888320=401408} 2024-09-21T10:03:32.653 INFO:tasks.rados.rados.0.smithi033.stdout:3817: writing smithi03385152-22 from 8421376 to 8888320 tid 1 2024-09-21T10:03:32.655 INFO:tasks.rados.rados.0.smithi033.stdout:3817: writing smithi03385152-22 from 8888320 to 9289728 tid 2 2024-09-21T10:03:32.655 INFO:tasks.rados.rados.0.smithi033.stdout:3818: read oid 37 snap -1 2024-09-21T10:03:32.655 INFO:tasks.rados.rados.0.smithi033.stdout:3818: expect deleted 2024-09-21T10:03:32.656 INFO:tasks.rados.rados.0.smithi033.stdout:3819: read oid 40 snap -1 2024-09-21T10:03:32.656 INFO:tasks.rados.rados.0.smithi033.stdout:3819: expect (ObjNum 1174 snap 336 seq_num 1174) 2024-09-21T10:03:32.656 INFO:tasks.rados.rados.0.smithi033.stdout:3820: snap_remove snap 343 2024-09-21T10:03:32.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:32 smithi033 ceph-mon[98546]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T10:03:32.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:32 smithi033 ceph-mon[96568]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T10:03:33.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:32 smithi105 ceph-mon[86649]: osdmap e773: 8 total, 8 up, 8 in 2024-09-21T10:03:33.582 INFO:tasks.rados.rados.0.smithi033.stdout:3820: done (4 left) 2024-09-21T10:03:33.582 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 2 current snap is 346 2024-09-21T10:03:33.582 INFO:tasks.rados.rados.0.smithi033.stdout:3821: seq_num 1199 ranges {8732672=483328,9216000=401408} 2024-09-21T10:03:33.634 INFO:tasks.rados.rados.0.smithi033.stdout:3821: writing smithi03385152-2 from 8732672 to 9216000 tid 1 2024-09-21T10:03:33.636 INFO:tasks.rados.rados.0.smithi033.stdout:3821: writing smithi03385152-2 from 9216000 to 9617408 tid 2 2024-09-21T10:03:33.636 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 10 current snap is 346 2024-09-21T10:03:33.636 INFO:tasks.rados.rados.0.smithi033.stdout:3822: seq_num 1200 ranges {2482176=499712,2981888=401408} 2024-09-21T10:03:33.652 INFO:tasks.rados.rados.0.smithi033.stdout:3822: writing smithi03385152-10 from 2482176 to 2981888 tid 1 2024-09-21T10:03:33.655 INFO:tasks.rados.rados.0.smithi033.stdout:3822: writing smithi03385152-10 from 2981888 to 3383296 tid 2 2024-09-21T10:03:33.655 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 346 2024-09-21T10:03:33.655 INFO:tasks.rados.rados.0.smithi033.stdout:3823: seq_num 1201 ranges {2334720=524288,2859008=401408} 2024-09-21T10:03:33.670 INFO:tasks.rados.rados.0.smithi033.stdout:3823: writing smithi03385152-18 from 2334720 to 2859008 tid 1 2024-09-21T10:03:33.673 INFO:tasks.rados.rados.0.smithi033.stdout:3823: writing smithi03385152-18 from 2859008 to 3260416 tid 2 2024-09-21T10:03:33.673 INFO:tasks.rados.rados.0.smithi033.stdout:3824: snap_remove snap 345 2024-09-21T10:03:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:33 smithi033 ceph-mon[98546]: pgmap v795: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 4.2 MiB/s wr, 20 op/s 2024-09-21T10:03:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:33 smithi033 ceph-mon[98546]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T10:03:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:33 smithi033 ceph-mon[98546]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T10:03:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:33 smithi033 ceph-mon[96568]: pgmap v795: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 4.2 MiB/s wr, 20 op/s 2024-09-21T10:03:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:33 smithi033 ceph-mon[96568]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T10:03:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:33 smithi033 ceph-mon[96568]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T10:03:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:33 smithi105 ceph-mon[86649]: pgmap v795: 121 pgs: 1 active+clean+snaptrim_wait, 120 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 4.2 MiB/s wr, 20 op/s 2024-09-21T10:03:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:33 smithi105 ceph-mon[86649]: osdmap e774: 8 total, 8 up, 8 in 2024-09-21T10:03:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:33 smithi105 ceph-mon[86649]: osdmap e775: 8 total, 8 up, 8 in 2024-09-21T10:03:34.609 INFO:tasks.rados.rados.0.smithi033.stdout:3824: done (7 left) 2024-09-21T10:03:34.609 INFO:tasks.rados.rados.0.smithi033.stdout:3825: rmattr oid 5 current snap is 346 2024-09-21T10:03:34.609 INFO:tasks.rados.rados.0.smithi033.stdout:3816: expect (ObjNum 1196 snap 345 seq_num 1196) 2024-09-21T10:03:34.640 INFO:tasks.rados.rados.0.smithi033.stdout:3817: finishing write tid 1 to smithi03385152-22 2024-09-21T10:03:34.640 INFO:tasks.rados.rados.0.smithi033.stdout:3817: finishing write tid 2 to smithi03385152-22 2024-09-21T10:03:34.640 INFO:tasks.rados.rados.0.smithi033.stdout:3817: finishing write tid 3 to smithi03385152-22 2024-09-21T10:03:34.640 INFO:tasks.rados.rados.0.smithi033.stdout:3819: expect (ObjNum 1174 snap 336 seq_num 1174) 2024-09-21T10:03:34.724 INFO:tasks.rados.rados.0.smithi033.stdout:3817: finishing write tid 4 to smithi03385152-22 2024-09-21T10:03:34.724 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2493 (ObjNum 1198 snap 346 seq_num 1198) dirty exists 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3817: left oid 22 (ObjNum 1198 snap 346 seq_num 1198) 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3816: done (7 left) 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3817: done (6 left) 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3818: done (5 left) 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3819: done (4 left) 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3826: read oid 9 snap -1 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3826: expect (ObjNum 1189 snap 344 seq_num 1189) 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3827: read oid 38 snap -1 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3827: expect (ObjNum 1183 snap 340 seq_num 1183) 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3828: setattr oid 12 current snap is 346 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3821: finishing write tid 1 to smithi03385152-2 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3821: finishing write tid 2 to smithi03385152-2 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3821: finishing write tid 3 to smithi03385152-2 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3821: finishing write tid 4 to smithi03385152-2 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 2412 (ObjNum 1199 snap 346 seq_num 1199) dirty exists 2024-09-21T10:03:34.725 INFO:tasks.rados.rados.0.smithi033.stdout:3821: left oid 2 (ObjNum 1199 snap 346 seq_num 1199) 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3822: finishing write tid 1 to smithi03385152-10 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3822: finishing write tid 2 to smithi03385152-10 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3822: finishing write tid 3 to smithi03385152-10 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3822: finishing write tid 4 to smithi03385152-10 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 2494 (ObjNum 1200 snap 346 seq_num 1200) dirty exists 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3822: left oid 10 (ObjNum 1200 snap 346 seq_num 1200) 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3823: finishing write tid 1 to smithi03385152-18 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3823: finishing write tid 2 to smithi03385152-18 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3823: finishing write tid 3 to smithi03385152-18 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3823: finishing write tid 4 to smithi03385152-18 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1835 (ObjNum 1201 snap 346 seq_num 1201) dirty exists 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:3823: left oid 18 (ObjNum 1201 snap 346 seq_num 1201) 2024-09-21T10:03:34.726 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2421 (ObjNum 1157 snap 331 seq_num 1157) dirty exists 2024-09-21T10:03:34.727 INFO:tasks.rados.rados.0.smithi033.stdout:3821: done (6 left) 2024-09-21T10:03:34.727 INFO:tasks.rados.rados.0.smithi033.stdout:3822: done (5 left) 2024-09-21T10:03:34.727 INFO:tasks.rados.rados.0.smithi033.stdout:3823: done (4 left) 2024-09-21T10:03:34.727 INFO:tasks.rados.rados.0.smithi033.stdout:3825: done (3 left) 2024-09-21T10:03:34.727 INFO:tasks.rados.rados.0.smithi033.stdout:3829: setattr oid 14 current snap is 346 2024-09-21T10:03:34.727 INFO:tasks.rados.rados.0.smithi033.stdout:3830: snap_create 2024-09-21T10:03:34.729 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2496 (ObjNum 1181 snap 338 seq_num 1181) dirty exists 2024-09-21T10:03:34.730 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2494 (ObjNum 1001 snap 292 seq_num 1001) dirty exists 2024-09-21T10:03:34.733 INFO:tasks.rados.rados.0.smithi033.stdout:3826: expect (ObjNum 1189 snap 344 seq_num 1189) 2024-09-21T10:03:34.892 INFO:tasks.rados.rados.0.smithi033.stdout:3827: expect (ObjNum 1183 snap 340 seq_num 1183) 2024-09-21T10:03:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3826: done (4 left) 2024-09-21T10:03:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3827: done (3 left) 2024-09-21T10:03:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3828: done (2 left) 2024-09-21T10:03:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3829: done (1 left) 2024-09-21T10:03:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3830: done (0 left) 2024-09-21T10:03:35.615 INFO:tasks.rados.rados.0.smithi033.stdout:3831: snap_create 2024-09-21T10:03:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:35 smithi033 ceph-mon[98546]: pgmap v798: 121 pgs: 121 active+clean; 148 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T10:03:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:35 smithi033 ceph-mon[98546]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T10:03:35.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:35 smithi033 ceph-mon[96568]: pgmap v798: 121 pgs: 121 active+clean; 148 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T10:03:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:35 smithi033 ceph-mon[96568]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T10:03:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:35 smithi105 ceph-mon[86649]: pgmap v798: 121 pgs: 121 active+clean; 148 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-21T10:03:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:35 smithi105 ceph-mon[86649]: osdmap e776: 8 total, 8 up, 8 in 2024-09-21T10:03:36.623 INFO:tasks.rados.rados.0.smithi033.stdout:3831: done (0 left) 2024-09-21T10:03:36.623 INFO:tasks.rados.rados.0.smithi033.stdout:3832: read oid 11 snap -1 2024-09-21T10:03:36.623 INFO:tasks.rados.rados.0.smithi033.stdout:3832: expect (ObjNum 1197 snap 345 seq_num 1197) 2024-09-21T10:03:36.623 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 348 2024-09-21T10:03:36.623 INFO:tasks.rados.rados.0.smithi033.stdout:3833: seq_num 1202 ranges {5120000=540672,5660672=401408} 2024-09-21T10:03:36.656 INFO:tasks.rados.rados.0.smithi033.stdout:3833: writing smithi03385152-28 from 5120000 to 5660672 tid 1 2024-09-21T10:03:36.658 INFO:tasks.rados.rados.0.smithi033.stdout:3833: writing smithi03385152-28 from 5660672 to 6062080 tid 2 2024-09-21T10:03:36.658 INFO:tasks.rados.rados.0.smithi033.stdout:3834: delete oid 46 current snap is 348 2024-09-21T10:03:36.658 INFO:tasks.rados.rados.0.smithi033.stdout:3832: expect (ObjNum 1197 snap 345 seq_num 1197) 2024-09-21T10:03:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:36 smithi033 ceph-mon[98546]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T10:03:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:36 smithi033 ceph-mon[96568]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T10:03:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:36 smithi105 ceph-mon[86649]: osdmap e777: 8 total, 8 up, 8 in 2024-09-21T10:03:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:3833: finishing write tid 1 to smithi03385152-28 2024-09-21T10:03:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:3833: finishing write tid 2 to smithi03385152-28 2024-09-21T10:03:37.482 INFO:tasks.rados.rados.0.smithi033.stdout:3832: done (2 left) 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3834: done (1 left) 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3835: read oid 44 snap -1 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3835: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3836: rollback oid 38 current snap is 348 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 38 to 346 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3833: finishing write tid 3 to smithi03385152-28 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3833: finishing write tid 4 to smithi03385152-28 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1838 (ObjNum 1202 snap 348 seq_num 1202) dirty exists 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3833: left oid 28 (ObjNum 1202 snap 348 seq_num 1202) 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3833: done (2 left) 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3837: rmattr oid 39 current snap is 348 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3838: read oid 42 snap -1 2024-09-21T10:03:37.483 INFO:tasks.rados.rados.0.smithi033.stdout:3838: expect (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:03:37.484 INFO:tasks.rados.rados.0.smithi033.stdout:3839: snap_create 2024-09-21T10:03:37.485 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 39 v 2422 (ObjNum 1113 snap 324 seq_num 1113) dirty exists 2024-09-21T10:03:37.487 INFO:tasks.rados.rados.0.smithi033.stdout:3836: finishing rollback tid 0 to smithi03385152-38 2024-09-21T10:03:37.487 INFO:tasks.rados.rados.0.smithi033.stdout:3836: finishing rollback tid 1 to smithi03385152-38 2024-09-21T10:03:37.491 INFO:tasks.rados.rados.0.smithi033.stdout:3835: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:37.551 INFO:tasks.rados.rados.0.smithi033.stdout:3836: finishing rollback tid 2 to smithi03385152-38 2024-09-21T10:03:37.552 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2490 (ObjNum 1183 snap 340 seq_num 1183) dirty exists 2024-09-21T10:03:37.552 INFO:tasks.rados.rados.0.smithi033.stdout:3838: expect (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:03:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:37 smithi033 ceph-mon[98546]: pgmap v801: 121 pgs: 121 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 832 KiB/s wr, 17 op/s 2024-09-21T10:03:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:37 smithi033 ceph-mon[98546]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T10:03:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:37 smithi033 ceph-mon[96568]: pgmap v801: 121 pgs: 121 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 832 KiB/s wr, 17 op/s 2024-09-21T10:03:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:37 smithi033 ceph-mon[96568]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T10:03:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:37 smithi105 ceph-mon[86649]: pgmap v801: 121 pgs: 121 active+clean; 147 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 832 KiB/s wr, 17 op/s 2024-09-21T10:03:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:37 smithi105 ceph-mon[86649]: osdmap e778: 8 total, 8 up, 8 in 2024-09-21T10:03:38.195 INFO:tasks.rados.rados.0.smithi033.stdout:3835: done (4 left) 2024-09-21T10:03:38.195 INFO:tasks.rados.rados.0.smithi033.stdout:3836: done (3 left) 2024-09-21T10:03:38.195 INFO:tasks.rados.rados.0.smithi033.stdout:3837: done (2 left) 2024-09-21T10:03:38.195 INFO:tasks.rados.rados.0.smithi033.stdout:3838: done (1 left) 2024-09-21T10:03:38.195 INFO:tasks.rados.rados.0.smithi033.stdout:3839: done (0 left) 2024-09-21T10:03:38.195 INFO:tasks.rados.rados.0.smithi033.stdout:3840: delete oid 27 current snap is 349 2024-09-21T10:03:38.199 INFO:tasks.rados.rados.0.smithi033.stdout:3840: done (0 left) 2024-09-21T10:03:38.199 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 349 2024-09-21T10:03:38.199 INFO:tasks.rados.rados.0.smithi033.stdout:3841: seq_num 1205 ranges {0=589824,589824=401408} 2024-09-21T10:03:38.202 INFO:tasks.rados.rados.0.smithi033.stdout:3841: writing smithi03385152-27 from 0 to 589824 tid 1 2024-09-21T10:03:38.204 INFO:tasks.rados.rados.0.smithi033.stdout:3841: writing smithi03385152-27 from 589824 to 991232 tid 2 2024-09-21T10:03:38.205 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 349 2024-09-21T10:03:38.205 INFO:tasks.rados.rados.0.smithi033.stdout:3842: seq_num 1206 ranges {0=606208,606208=401408} 2024-09-21T10:03:38.208 INFO:tasks.rados.rados.0.smithi033.stdout:3842: writing smithi03385152-36 from 0 to 606208 tid 1 2024-09-21T10:03:38.210 INFO:tasks.rados.rados.0.smithi033.stdout:3842: writing smithi03385152-36 from 606208 to 1007616 tid 2 2024-09-21T10:03:38.210 INFO:tasks.rados.rados.0.smithi033.stdout:3843: delete oid 13 current snap is 349 2024-09-21T10:03:38.210 INFO:tasks.rados.rados.0.smithi033.stdout:3841: finishing write tid 1 to smithi03385152-27 2024-09-21T10:03:38.211 INFO:tasks.rados.rados.0.smithi033.stdout:3841: finishing write tid 2 to smithi03385152-27 2024-09-21T10:03:38.211 INFO:tasks.rados.rados.0.smithi033.stdout:3841: finishing write tid 3 to smithi03385152-27 2024-09-21T10:03:38.211 INFO:tasks.rados.rados.0.smithi033.stdout:3841: finishing write tid 4 to smithi03385152-27 2024-09-21T10:03:38.211 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2503 (ObjNum 1205 snap 349 seq_num 1205) dirty exists 2024-09-21T10:03:38.211 INFO:tasks.rados.rados.0.smithi033.stdout:3841: left oid 27 (ObjNum 1205 snap 349 seq_num 1205) 2024-09-21T10:03:38.214 INFO:tasks.rados.rados.0.smithi033.stdout:3842: finishing write tid 1 to smithi03385152-36 2024-09-21T10:03:38.215 INFO:tasks.rados.rados.0.smithi033.stdout:3842: finishing write tid 2 to smithi03385152-36 2024-09-21T10:03:38.215 INFO:tasks.rados.rados.0.smithi033.stdout:3842: finishing write tid 3 to smithi03385152-36 2024-09-21T10:03:38.216 INFO:tasks.rados.rados.0.smithi033.stdout:3841: done (2 left) 2024-09-21T10:03:38.216 INFO:tasks.rados.rados.0.smithi033.stdout:3843: done (1 left) 2024-09-21T10:03:38.216 INFO:tasks.rados.rados.0.smithi033.stdout:3844: read oid 6 snap -1 2024-09-21T10:03:38.216 INFO:tasks.rados.rados.0.smithi033.stdout:3844: expect (ObjNum 1146 snap 329 seq_num 1146) 2024-09-21T10:03:38.216 INFO:tasks.rados.rados.0.smithi033.stdout:3845: read oid 27 snap -1 2024-09-21T10:03:38.217 INFO:tasks.rados.rados.0.smithi033.stdout:3845: expect (ObjNum 1205 snap 349 seq_num 1205) 2024-09-21T10:03:38.217 INFO:tasks.rados.rados.0.smithi033.stdout:3846: delete oid 3 current snap is 349 2024-09-21T10:03:38.218 INFO:tasks.rados.rados.0.smithi033.stdout:3842: finishing write tid 4 to smithi03385152-36 2024-09-21T10:03:38.218 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2750 (ObjNum 1206 snap 349 seq_num 1206) dirty exists 2024-09-21T10:03:38.218 INFO:tasks.rados.rados.0.smithi033.stdout:3842: left oid 36 (ObjNum 1206 snap 349 seq_num 1206) 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:3842: done (3 left) 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:3846: done (2 left) 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:3847: copy_from oid 3 from oid 17 current snap is 349 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:3848: copy_from oid 45 from oid 15 current snap is 349 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:3849: rollback oid 47 current snap is 349 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 47 to 348 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:3850: copy_from oid 38 from oid 28 current snap is 349 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:3851: read oid 42 snap -1 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:3851: expect (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:03:38.220 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 34 current snap is 349 2024-09-21T10:03:38.221 INFO:tasks.rados.rados.0.smithi033.stdout:3852: seq_num 1209 ranges {696320=655360,1351680=401408} 2024-09-21T10:03:38.228 INFO:tasks.rados.rados.0.smithi033.stdout:3852: writing smithi03385152-34 from 696320 to 1351680 tid 1 2024-09-21T10:03:38.230 INFO:tasks.rados.rados.0.smithi033.stdout:3852: writing smithi03385152-34 from 1351680 to 1753088 tid 2 2024-09-21T10:03:38.230 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 46 current snap is 349 2024-09-21T10:03:38.230 INFO:tasks.rados.rados.0.smithi033.stdout:3849: finishing rollback tid 1 to smithi03385152-47 2024-09-21T10:03:38.230 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 0 (ObjNum 289 snap 0 seq_num 1) dirty dne 2024-09-21T10:03:38.230 INFO:tasks.rados.rados.0.smithi033.stdout:3853: seq_num 1210 ranges {0=671744,671744=401408} 2024-09-21T10:03:38.234 INFO:tasks.rados.rados.0.smithi033.stdout:3853: writing smithi03385152-46 from 0 to 671744 tid 1 2024-09-21T10:03:38.236 INFO:tasks.rados.rados.0.smithi033.stdout:3853: writing smithi03385152-46 from 671744 to 1073152 tid 2 2024-09-21T10:03:38.236 INFO:tasks.rados.rados.0.smithi033.stdout:3849: done (8 left) 2024-09-21T10:03:38.236 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 18 current snap is 349 2024-09-21T10:03:38.237 INFO:tasks.rados.rados.0.smithi033.stdout:3854: seq_num 1211 ranges {3260416=688128,3948544=401408} 2024-09-21T10:03:38.259 INFO:tasks.rados.rados.0.smithi033.stdout:3854: writing smithi03385152-18 from 3260416 to 3948544 tid 1 2024-09-21T10:03:38.261 INFO:tasks.rados.rados.0.smithi033.stdout:3854: writing smithi03385152-18 from 3948544 to 4349952 tid 2 2024-09-21T10:03:38.261 INFO:tasks.rados.rados.0.smithi033.stdout:3855: delete oid 2 current snap is 349 2024-09-21T10:03:38.261 INFO:tasks.rados.rados.0.smithi033.stdout:3844: expect (ObjNum 1146 snap 329 seq_num 1146) 2024-09-21T10:03:38.326 INFO:tasks.rados.rados.0.smithi033.stdout:3845: expect (ObjNum 1205 snap 349 seq_num 1205) 2024-09-21T10:03:38.352 INFO:tasks.rados.rados.0.smithi033.stdout:3848: finishing copy_from racing read to smithi03385152-45 2024-09-21T10:03:38.352 INFO:tasks.rados.rados.0.smithi033.stdout:3847: finishing copy_from racing read to smithi03385152-3 2024-09-21T10:03:38.352 INFO:tasks.rados.rados.0.smithi033.stdout:3847: finishing copy_from to smithi03385152-3 2024-09-21T10:03:38.352 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2495 (ObjNum 1192 snap 345 seq_num 1192) dirty exists 2024-09-21T10:03:38.352 INFO:tasks.rados.rados.0.smithi033.stdout:3848: finishing copy_from to smithi03385152-45 2024-09-21T10:03:38.352 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 1642 (ObjNum 980 snap 285 seq_num 980) dirty exists 2024-09-21T10:03:38.352 INFO:tasks.rados.rados.0.smithi033.stdout:3852: finishing write tid 1 to smithi03385152-34 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3844: done (9 left) 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3845: done (8 left) 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3847: done (7 left) 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3848: done (6 left) 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3855: done (5 left) 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3856: snap_create 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3852: finishing write tid 2 to smithi03385152-34 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3852: finishing write tid 3 to smithi03385152-34 2024-09-21T10:03:38.353 INFO:tasks.rados.rados.0.smithi033.stdout:3851: expect (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:03:38.995 INFO:tasks.rados.rados.0.smithi033.stdout:3853: finishing write tid 1 to smithi03385152-46 2024-09-21T10:03:38.995 INFO:tasks.rados.rados.0.smithi033.stdout:3852: finishing write tid 4 to smithi03385152-34 2024-09-21T10:03:38.995 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 34 v 2499 (ObjNum 1209 snap 349 seq_num 1209) dirty exists 2024-09-21T10:03:38.995 INFO:tasks.rados.rados.0.smithi033.stdout:3852: left oid 34 (ObjNum 1209 snap 349 seq_num 1209) 2024-09-21T10:03:38.995 INFO:tasks.rados.rados.0.smithi033.stdout:3853: finishing write tid 2 to smithi03385152-46 2024-09-21T10:03:38.995 INFO:tasks.rados.rados.0.smithi033.stdout:3853: finishing write tid 3 to smithi03385152-46 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3853: finishing write tid 4 to smithi03385152-46 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 46 v 2502 (ObjNum 1210 snap 349 seq_num 1210) dirty exists 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3853: left oid 46 (ObjNum 1210 snap 349 seq_num 1210) 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3850: finishing copy_from racing read to smithi03385152-38 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3851: done (5 left) 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3852: done (4 left) 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3853: done (3 left) 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3856: done (2 left) 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 350 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3850: finishing copy_from to smithi03385152-38 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 1838 (ObjNum 1202 snap 348 seq_num 1202) dirty exists 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3854: finishing write tid 1 to smithi03385152-18 2024-09-21T10:03:38.996 INFO:tasks.rados.rados.0.smithi033.stdout:3857: seq_num 1213 ranges {9289728=720896,10010624=401408} 2024-09-21T10:03:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:38 smithi033 ceph-mon[98546]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T10:03:38.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:38 smithi033 ceph-mon[98546]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T10:03:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:38 smithi033 ceph-mon[96568]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T10:03:38.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:38 smithi033 ceph-mon[96568]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T10:03:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:38 smithi105 ceph-mon[86649]: osdmap e779: 8 total, 8 up, 8 in 2024-09-21T10:03:39.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:38 smithi105 ceph-mon[86649]: osdmap e780: 8 total, 8 up, 8 in 2024-09-21T10:03:39.049 INFO:tasks.rados.rados.0.smithi033.stdout:3857: writing smithi03385152-22 from 9289728 to 10010624 tid 1 2024-09-21T10:03:39.052 INFO:tasks.rados.rados.0.smithi033.stdout:3857: writing smithi03385152-22 from 10010624 to 10412032 tid 2 2024-09-21T10:03:39.052 INFO:tasks.rados.rados.0.smithi033.stdout:3850: done (2 left) 2024-09-21T10:03:39.052 INFO:tasks.rados.rados.0.smithi033.stdout:3858: read oid 16 snap -1 2024-09-21T10:03:39.052 INFO:tasks.rados.rados.0.smithi033.stdout:3858: expect (ObjNum 1193 snap 345 seq_num 1193) 2024-09-21T10:03:39.052 INFO:tasks.rados.rados.0.smithi033.stdout:3859: snap_create 2024-09-21T10:03:39.052 INFO:tasks.rados.rados.0.smithi033.stdout:3854: finishing write tid 2 to smithi03385152-18 2024-09-21T10:03:39.052 INFO:tasks.rados.rados.0.smithi033.stdout:3854: finishing write tid 3 to smithi03385152-18 2024-09-21T10:03:39.052 INFO:tasks.rados.rados.0.smithi033.stdout:3854: finishing write tid 4 to smithi03385152-18 2024-09-21T10:03:39.053 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 18 v 1838 (ObjNum 1211 snap 349 seq_num 1211) dirty exists 2024-09-21T10:03:39.053 INFO:tasks.rados.rados.0.smithi033.stdout:3854: left oid 18 (ObjNum 1211 snap 349 seq_num 1211) 2024-09-21T10:03:39.059 INFO:tasks.rados.rados.0.smithi033.stdout:3857: finishing write tid 1 to smithi03385152-22 2024-09-21T10:03:39.059 INFO:tasks.rados.rados.0.smithi033.stdout:3857: finishing write tid 2 to smithi03385152-22 2024-09-21T10:03:39.059 INFO:tasks.rados.rados.0.smithi033.stdout:3857: finishing write tid 3 to smithi03385152-22 2024-09-21T10:03:39.061 INFO:tasks.rados.rados.0.smithi033.stdout:3857: finishing write tid 4 to smithi03385152-22 2024-09-21T10:03:39.061 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2505 (ObjNum 1213 snap 350 seq_num 1213) dirty exists 2024-09-21T10:03:39.062 INFO:tasks.rados.rados.0.smithi033.stdout:3857: left oid 22 (ObjNum 1213 snap 350 seq_num 1213) 2024-09-21T10:03:39.065 INFO:tasks.rados.rados.0.smithi033.stdout:3858: expect (ObjNum 1193 snap 345 seq_num 1193) 2024-09-21T10:03:39.641 INFO:tasks.rados.rados.0.smithi033.stdout:3854: done (3 left) 2024-09-21T10:03:39.641 INFO:tasks.rados.rados.0.smithi033.stdout:3857: done (2 left) 2024-09-21T10:03:39.641 INFO:tasks.rados.rados.0.smithi033.stdout:3858: done (1 left) 2024-09-21T10:03:39.641 INFO:tasks.rados.rados.0.smithi033.stdout:3859: done (0 left) 2024-09-21T10:03:39.641 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 28 current snap is 351 2024-09-21T10:03:39.641 INFO:tasks.rados.rados.0.smithi033.stdout:3860: seq_num 1214 ranges {6062080=737280,6799360=401408} 2024-09-21T10:03:39.678 INFO:tasks.rados.rados.0.smithi033.stdout:3860: writing smithi03385152-28 from 6062080 to 6799360 tid 1 2024-09-21T10:03:39.680 INFO:tasks.rados.rados.0.smithi033.stdout:3860: writing smithi03385152-28 from 6799360 to 7200768 tid 2 2024-09-21T10:03:39.680 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 20 current snap is 351 2024-09-21T10:03:39.680 INFO:tasks.rados.rados.0.smithi033.stdout:3861: seq_num 1215 ranges {335872=753664,1089536=401408} 2024-09-21T10:03:39.686 INFO:tasks.rados.rados.0.smithi033.stdout:3861: writing smithi03385152-20 from 335872 to 1089536 tid 1 2024-09-21T10:03:39.689 INFO:tasks.rados.rados.0.smithi033.stdout:3861: writing smithi03385152-20 from 1089536 to 1490944 tid 2 2024-09-21T10:03:39.689 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 40 current snap is 351 2024-09-21T10:03:39.689 INFO:tasks.rados.rados.0.smithi033.stdout:3860: finishing write tid 1 to smithi03385152-28 2024-09-21T10:03:39.689 INFO:tasks.rados.rados.0.smithi033.stdout:3860: finishing write tid 2 to smithi03385152-28 2024-09-21T10:03:39.689 INFO:tasks.rados.rados.0.smithi033.stdout:3860: finishing write tid 3 to smithi03385152-28 2024-09-21T10:03:39.689 INFO:tasks.rados.rados.0.smithi033.stdout:3862: seq_num 1216 ranges {2146304=770048,2916352=401408} 2024-09-21T10:03:39.704 INFO:tasks.rados.rados.0.smithi033.stdout:3862: writing smithi03385152-40 from 2146304 to 2916352 tid 1 2024-09-21T10:03:39.707 INFO:tasks.rados.rados.0.smithi033.stdout:3862: writing smithi03385152-40 from 2916352 to 3317760 tid 2 2024-09-21T10:03:39.707 INFO:tasks.rados.rados.0.smithi033.stdout:3863: setattr oid 27 current snap is 351 2024-09-21T10:03:39.708 INFO:tasks.rados.rados.0.smithi033.stdout:3860: finishing write tid 4 to smithi03385152-28 2024-09-21T10:03:39.708 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1841 (ObjNum 1214 snap 351 seq_num 1214) dirty exists 2024-09-21T10:03:39.708 INFO:tasks.rados.rados.0.smithi033.stdout:3860: left oid 28 (ObjNum 1214 snap 351 seq_num 1214) 2024-09-21T10:03:39.708 INFO:tasks.rados.rados.0.smithi033.stdout:3861: finishing write tid 1 to smithi03385152-20 2024-09-21T10:03:39.708 INFO:tasks.rados.rados.0.smithi033.stdout:3861: finishing write tid 2 to smithi03385152-20 2024-09-21T10:03:39.708 INFO:tasks.rados.rados.0.smithi033.stdout:3861: finishing write tid 3 to smithi03385152-20 2024-09-21T10:03:39.709 INFO:tasks.rados.rados.0.smithi033.stdout:3861: finishing write tid 4 to smithi03385152-20 2024-09-21T10:03:39.709 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 2469 (ObjNum 1215 snap 351 seq_num 1215) dirty exists 2024-09-21T10:03:39.709 INFO:tasks.rados.rados.0.smithi033.stdout:3861: left oid 20 (ObjNum 1215 snap 351 seq_num 1215) 2024-09-21T10:03:39.709 INFO:tasks.rados.rados.0.smithi033.stdout:3860: done (3 left) 2024-09-21T10:03:39.710 INFO:tasks.rados.rados.0.smithi033.stdout:3861: done (2 left) 2024-09-21T10:03:39.710 INFO:tasks.rados.rados.0.smithi033.stdout:3864: delete oid 18 current snap is 351 2024-09-21T10:03:39.712 INFO:tasks.rados.rados.0.smithi033.stdout:3862: finishing write tid 1 to smithi03385152-40 2024-09-21T10:03:39.712 INFO:tasks.rados.rados.0.smithi033.stdout:3862: finishing write tid 2 to smithi03385152-40 2024-09-21T10:03:39.713 INFO:tasks.rados.rados.0.smithi033.stdout:3864: done (2 left) 2024-09-21T10:03:39.713 INFO:tasks.rados.rados.0.smithi033.stdout:3865: delete oid 41 current snap is 351 2024-09-21T10:03:39.713 INFO:tasks.rados.rados.0.smithi033.stdout:3862: finishing write tid 3 to smithi03385152-40 2024-09-21T10:03:39.713 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2504 (ObjNum 1205 snap 349 seq_num 1205) dirty exists 2024-09-21T10:03:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3862: finishing write tid 4 to smithi03385152-40 2024-09-21T10:03:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 40 v 2493 (ObjNum 1216 snap 351 seq_num 1216) dirty exists 2024-09-21T10:03:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3862: left oid 40 (ObjNum 1216 snap 351 seq_num 1216) 2024-09-21T10:03:39.716 INFO:tasks.rados.rados.0.smithi033.stdout:3862: done (2 left) 2024-09-21T10:03:39.717 INFO:tasks.rados.rados.0.smithi033.stdout:3863: done (1 left) 2024-09-21T10:03:39.717 INFO:tasks.rados.rados.0.smithi033.stdout:3865: done (0 left) 2024-09-21T10:03:39.717 INFO:tasks.rados.rados.0.smithi033.stdout:3866: rollback oid 17 current snap is 351 2024-09-21T10:03:39.717 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 348 2024-09-21T10:03:39.717 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 16 current snap is 351 2024-09-21T10:03:39.717 INFO:tasks.rados.rados.0.smithi033.stdout:3867: seq_num 1219 ranges {1073152=417792,1490944=557056,2048000=245760} 2024-09-21T10:03:39.724 INFO:tasks.rados.rados.0.smithi033.stdout:3867: writing smithi03385152-16 from 1073152 to 1490944 tid 1 2024-09-21T10:03:39.728 INFO:tasks.rados.rados.0.smithi033.stdout:3867: writing smithi03385152-16 from 1490944 to 2048000 tid 2 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3867: writing smithi03385152-16 from 2048000 to 2293760 tid 3 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3868: copy_from oid 14 from oid 31 current snap is 351 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3866: finishing rollback tid 0 to smithi03385152-17 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3866: finishing rollback tid 1 to smithi03385152-17 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3866: finishing rollback tid 2 to smithi03385152-17 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2499 (ObjNum 1192 snap 345 seq_num 1192) dirty exists 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3867: finishing write tid 1 to smithi03385152-16 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3866: done (2 left) 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3869: read oid 42 snap -1 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3869: expect (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:03:39.730 INFO:tasks.rados.rados.0.smithi033.stdout:3870: snap_remove snap 347 2024-09-21T10:03:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:39 smithi033 ceph-mon[98546]: pgmap v804: 121 pgs: 121 active+clean; 148 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 697 KiB/s rd, 862 KiB/s wr, 9 op/s 2024-09-21T10:03:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:39 smithi033 ceph-mon[96568]: pgmap v804: 121 pgs: 121 active+clean; 148 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 697 KiB/s rd, 862 KiB/s wr, 9 op/s 2024-09-21T10:03:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:39 smithi105 ceph-mon[86649]: pgmap v804: 121 pgs: 121 active+clean; 148 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 697 KiB/s rd, 862 KiB/s wr, 9 op/s 2024-09-21T10:03:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:03:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:03:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:40.656 INFO:tasks.rados.rados.0.smithi033.stdout:3870: done (3 left) 2024-09-21T10:03:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 47 current snap is 351 2024-09-21T10:03:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3867: finishing write tid 2 to smithi03385152-16 2024-09-21T10:03:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3867: finishing write tid 3 to smithi03385152-16 2024-09-21T10:03:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3867: finishing write tid 4 to smithi03385152-16 2024-09-21T10:03:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3867: finishing write tid 5 to smithi03385152-16 2024-09-21T10:03:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2757 (ObjNum 1219 snap 351 seq_num 1219) dirty exists 2024-09-21T10:03:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3867: left oid 16 (ObjNum 1219 snap 351 seq_num 1219) 2024-09-21T10:03:40.657 INFO:tasks.rados.rados.0.smithi033.stdout:3869: expect (ObjNum 1136 snap 327 seq_num 1136) 2024-09-21T10:03:40.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:40 smithi033 ceph-mon[98546]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T10:03:40.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:40 smithi033 ceph-mon[96568]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T10:03:41.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:40 smithi105 ceph-mon[86649]: osdmap e781: 8 total, 8 up, 8 in 2024-09-21T10:03:41.305 INFO:tasks.rados.rados.0.smithi033.stdout:3868: finishing copy_from racing read to smithi03385152-14 2024-09-21T10:03:41.305 INFO:tasks.rados.rados.0.smithi033.stdout:3868: finishing copy_from to smithi03385152-14 2024-09-21T10:03:41.305 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2588 (ObjNum 1067 snap 313 seq_num 1067) dirty exists 2024-09-21T10:03:41.305 INFO:tasks.rados.rados.0.smithi033.stdout:3871: seq_num 1220 ranges {0=434176,434176=442368,876544=360448} 2024-09-21T10:03:41.307 INFO:tasks.rados.rados.0.smithi033.stdout:3871: writing smithi03385152-47 from 0 to 434176 tid 1 2024-09-21T10:03:41.310 INFO:tasks.rados.rados.0.smithi033.stdout:3871: writing smithi03385152-47 from 434176 to 876544 tid 2 2024-09-21T10:03:41.313 INFO:tasks.rados.rados.0.smithi033.stdout:3871: writing smithi03385152-47 from 876544 to 1236992 tid 3 2024-09-21T10:03:41.313 INFO:tasks.rados.rados.0.smithi033.stdout:3867: done (3 left) 2024-09-21T10:03:41.313 INFO:tasks.rados.rados.0.smithi033.stdout:3868: done (2 left) 2024-09-21T10:03:41.314 INFO:tasks.rados.rados.0.smithi033.stdout:3869: done (1 left) 2024-09-21T10:03:41.314 INFO:tasks.rados.rados.0.smithi033.stdout:3872: read oid 15 snap 346 2024-09-21T10:03:41.314 INFO:tasks.rados.rados.0.smithi033.stdout:3872: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:03:41.314 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 13 current snap is 351 2024-09-21T10:03:41.314 INFO:tasks.rados.rados.0.smithi033.stdout:3873: seq_num 1221 ranges {0=458752,458752=729088,1187840=73728} 2024-09-21T10:03:41.315 INFO:tasks.rados.rados.0.smithi033.stdout:3873: writing smithi03385152-13 from 0 to 458752 tid 1 2024-09-21T10:03:41.320 INFO:tasks.rados.rados.0.smithi033.stdout:3873: writing smithi03385152-13 from 458752 to 1187840 tid 2 2024-09-21T10:03:41.322 INFO:tasks.rados.rados.0.smithi033.stdout:3873: writing smithi03385152-13 from 1187840 to 1261568 tid 3 2024-09-21T10:03:41.322 INFO:tasks.rados.rados.0.smithi033.stdout:3874: copy_from oid 10 from oid 28 current snap is 351 2024-09-21T10:03:41.322 INFO:tasks.rados.rados.0.smithi033.stdout:3871: finishing write tid 1 to smithi03385152-47 2024-09-21T10:03:41.322 INFO:tasks.rados.rados.0.smithi033.stdout:3871: finishing write tid 2 to smithi03385152-47 2024-09-21T10:03:41.323 INFO:tasks.rados.rados.0.smithi033.stdout:3871: finishing write tid 3 to smithi03385152-47 2024-09-21T10:03:41.323 INFO:tasks.rados.rados.0.smithi033.stdout:3871: finishing write tid 4 to smithi03385152-47 2024-09-21T10:03:41.323 INFO:tasks.rados.rados.0.smithi033.stdout:3871: finishing write tid 5 to smithi03385152-47 2024-09-21T10:03:41.323 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 1523 (ObjNum 1220 snap 351 seq_num 1220) dirty exists 2024-09-21T10:03:41.323 INFO:tasks.rados.rados.0.smithi033.stdout:3871: left oid 47 (ObjNum 1220 snap 351 seq_num 1220) 2024-09-21T10:03:41.323 INFO:tasks.rados.rados.0.smithi033.stdout:3871: done (3 left) 2024-09-21T10:03:41.323 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 351 2024-09-21T10:03:41.323 INFO:tasks.rados.rados.0.smithi033.stdout:3875: seq_num 1222 ranges {2318336=475136,2793472=606208,3399680=196608} 2024-09-21T10:03:41.336 INFO:tasks.rados.rados.0.smithi033.stdout:3875: writing smithi03385152-5 from 2318336 to 2793472 tid 1 2024-09-21T10:03:41.339 INFO:tasks.rados.rados.0.smithi033.stdout:3875: writing smithi03385152-5 from 2793472 to 3399680 tid 2 2024-09-21T10:03:41.341 INFO:tasks.rados.rados.0.smithi033.stdout:3875: writing smithi03385152-5 from 3399680 to 3596288 tid 3 2024-09-21T10:03:41.341 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 351 2024-09-21T10:03:41.341 INFO:tasks.rados.rados.0.smithi033.stdout:3876: seq_num 1223 ranges {12296192=491520,12787712=491520,13279232=311296} 2024-09-21T10:03:41.411 INFO:tasks.rados.rados.0.smithi033.stdout:3876: writing smithi03385152-31 from 12296192 to 12787712 tid 1 2024-09-21T10:03:41.413 INFO:tasks.rados.rados.0.smithi033.stdout:3876: writing smithi03385152-31 from 12787712 to 13279232 tid 2 2024-09-21T10:03:41.416 INFO:tasks.rados.rados.0.smithi033.stdout:3876: writing smithi03385152-31 from 13279232 to 13590528 tid 3 2024-09-21T10:03:41.416 INFO:tasks.rados.rados.0.smithi033.stdout:3877: read oid 46 snap 348 2024-09-21T10:03:41.416 INFO:tasks.rados.rados.0.smithi033.stdout:3877: expect deleted 2024-09-21T10:03:41.416 INFO:tasks.rados.rados.0.smithi033.stdout:3878: setattr oid 35 current snap is 351 2024-09-21T10:03:41.416 INFO:tasks.rados.rados.0.smithi033.stdout:3873: finishing write tid 1 to smithi03385152-13 2024-09-21T10:03:41.416 INFO:tasks.rados.rados.0.smithi033.stdout:3872: expect (ObjNum 980 snap 285 seq_num 980) 2024-09-21T10:03:41.474 INFO:tasks.rados.rados.0.smithi033.stdout:3873: finishing write tid 2 to smithi03385152-13 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3873: finishing write tid 3 to smithi03385152-13 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3873: finishing write tid 4 to smithi03385152-13 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3873: finishing write tid 5 to smithi03385152-13 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 13 v 2765 (ObjNum 1221 snap 351 seq_num 1221) dirty exists 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3873: left oid 13 (ObjNum 1221 snap 351 seq_num 1221) 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3875: finishing write tid 1 to smithi03385152-5 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3875: finishing write tid 2 to smithi03385152-5 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3875: finishing write tid 3 to smithi03385152-5 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3875: finishing write tid 4 to smithi03385152-5 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:3875: finishing write tid 5 to smithi03385152-5 2024-09-21T10:03:41.475 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2426 (ObjNum 1222 snap 351 seq_num 1222) dirty exists 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3875: left oid 5 (ObjNum 1222 snap 351 seq_num 1222) 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3874: finishing copy_from racing read to smithi03385152-10 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3874: finishing copy_from to smithi03385152-10 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 10 v 1841 (ObjNum 1214 snap 351 seq_num 1214) dirty exists 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3876: finishing write tid 1 to smithi03385152-31 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3876: finishing write tid 2 to smithi03385152-31 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3876: finishing write tid 3 to smithi03385152-31 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3876: finishing write tid 4 to smithi03385152-31 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3876: finishing write tid 5 to smithi03385152-31 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2770 (ObjNum 1223 snap 351 seq_num 1223) dirty exists 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3876: left oid 31 (ObjNum 1223 snap 351 seq_num 1223) 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3872: done (6 left) 2024-09-21T10:03:41.476 INFO:tasks.rados.rados.0.smithi033.stdout:3873: done (5 left) 2024-09-21T10:03:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:3874: done (4 left) 2024-09-21T10:03:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:3875: done (3 left) 2024-09-21T10:03:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:3876: done (2 left) 2024-09-21T10:03:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:3877: done (1 left) 2024-09-21T10:03:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:3879: delete oid 39 current snap is 351 2024-09-21T10:03:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:3879: done (1 left) 2024-09-21T10:03:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:3880: setattr oid 2 current snap is 351 2024-09-21T10:03:41.477 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 35 v 2771 (ObjNum 177 snap 0 seq_num 1163894144) dirty exists 2024-09-21T10:03:41.478 INFO:tasks.rados.rados.0.smithi033.stdout:3878: done (1 left) 2024-09-21T10:03:41.478 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 25 current snap is 351 2024-09-21T10:03:41.478 INFO:tasks.rados.rados.0.smithi033.stdout:3881: seq_num 1225 ranges {0=524288,524288=712704,1236992=90112} 2024-09-21T10:03:41.478 INFO:tasks.rados.rados.0.smithi033.stdout:3881: writing smithi03385152-25 from 0 to 524288 tid 1 2024-09-21T10:03:41.482 INFO:tasks.rados.rados.0.smithi033.stdout:3881: writing smithi03385152-25 from 524288 to 1236992 tid 2 2024-09-21T10:03:41.483 INFO:tasks.rados.rados.0.smithi033.stdout:3881: writing smithi03385152-25 from 1236992 to 1327104 tid 3 2024-09-21T10:03:41.483 INFO:tasks.rados.rados.0.smithi033.stdout:3882: read oid 38 snap -1 2024-09-21T10:03:41.483 INFO:tasks.rados.rados.0.smithi033.stdout:3882: expect (ObjNum 1202 snap 348 seq_num 1202) 2024-09-21T10:03:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3883: read oid 24 snap -1 2024-09-21T10:03:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3883: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:03:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 351 2024-09-21T10:03:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 2414 (ObjNum 0 snap 32666 seq_num 304) dirty exists 2024-09-21T10:03:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3881: finishing write tid 1 to smithi03385152-25 2024-09-21T10:03:41.484 INFO:tasks.rados.rados.0.smithi033.stdout:3884: seq_num 1226 ranges {2867200=540672,3407872=598016,4005888=204800} 2024-09-21T10:03:41.503 INFO:tasks.rados.rados.0.smithi033.stdout:3884: writing smithi03385152-21 from 2867200 to 3407872 tid 1 2024-09-21T10:03:41.506 INFO:tasks.rados.rados.0.smithi033.stdout:3884: writing smithi03385152-21 from 3407872 to 4005888 tid 2 2024-09-21T10:03:41.508 INFO:tasks.rados.rados.0.smithi033.stdout:3884: writing smithi03385152-21 from 4005888 to 4210688 tid 3 2024-09-21T10:03:41.508 INFO:tasks.rados.rados.0.smithi033.stdout:3880: done (4 left) 2024-09-21T10:03:41.508 INFO:tasks.rados.rados.0.smithi033.stdout:3885: read oid 12 snap -1 2024-09-21T10:03:41.508 INFO:tasks.rados.rados.0.smithi033.stdout:3885: expect (ObjNum 1181 snap 338 seq_num 1181) 2024-09-21T10:03:41.508 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 351 2024-09-21T10:03:41.509 INFO:tasks.rados.rados.0.smithi033.stdout:3886: seq_num 1227 ranges {991232=557056,1548288=483328,2031616=319488} 2024-09-21T10:03:41.517 INFO:tasks.rados.rados.0.smithi033.stdout:3886: writing smithi03385152-27 from 991232 to 1548288 tid 1 2024-09-21T10:03:41.519 INFO:tasks.rados.rados.0.smithi033.stdout:3886: writing smithi03385152-27 from 1548288 to 2031616 tid 2 2024-09-21T10:03:41.521 INFO:tasks.rados.rados.0.smithi033.stdout:3886: writing smithi03385152-27 from 2031616 to 2351104 tid 3 2024-09-21T10:03:41.521 INFO:tasks.rados.rados.0.smithi033.stdout:3887: copy_from oid 47 from oid 6 current snap is 351 2024-09-21T10:03:41.521 INFO:tasks.rados.rados.0.smithi033.stdout:3881: finishing write tid 2 to smithi03385152-25 2024-09-21T10:03:41.521 INFO:tasks.rados.rados.0.smithi033.stdout:3881: finishing write tid 3 to smithi03385152-25 2024-09-21T10:03:41.521 INFO:tasks.rados.rados.0.smithi033.stdout:3881: finishing write tid 4 to smithi03385152-25 2024-09-21T10:03:41.521 INFO:tasks.rados.rados.0.smithi033.stdout:3883: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:03:41.567 INFO:tasks.rados.rados.0.smithi033.stdout:3881: finishing write tid 5 to smithi03385152-25 2024-09-21T10:03:41.567 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 25 v 2418 (ObjNum 1225 snap 351 seq_num 1225) dirty exists 2024-09-21T10:03:41.568 INFO:tasks.rados.rados.0.smithi033.stdout:3881: left oid 25 (ObjNum 1225 snap 351 seq_num 1225) 2024-09-21T10:03:41.568 INFO:tasks.rados.rados.0.smithi033.stdout:3884: finishing write tid 1 to smithi03385152-21 2024-09-21T10:03:41.568 INFO:tasks.rados.rados.0.smithi033.stdout:3882: expect (ObjNum 1202 snap 348 seq_num 1202) 2024-09-21T10:03:41.957 INFO:tasks.rados.rados.0.smithi033.stdout:3884: finishing write tid 2 to smithi03385152-21 2024-09-21T10:03:41.957 INFO:tasks.rados.rados.0.smithi033.stdout:3884: finishing write tid 3 to smithi03385152-21 2024-09-21T10:03:41.958 INFO:tasks.rados.rados.0.smithi033.stdout:3881: done (6 left) 2024-09-21T10:03:41.958 INFO:tasks.rados.rados.0.smithi033.stdout:3882: done (5 left) 2024-09-21T10:03:41.958 INFO:tasks.rados.rados.0.smithi033.stdout:3883: done (4 left) 2024-09-21T10:03:41.958 INFO:tasks.rados.rados.0.smithi033.stdout:3888: read oid 44 snap -1 2024-09-21T10:03:41.958 INFO:tasks.rados.rados.0.smithi033.stdout:3888: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:41.958 INFO:tasks.rados.rados.0.smithi033.stdout:3889: snap_remove snap 349 2024-09-21T10:03:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:41 smithi033 ceph-mon[96568]: pgmap v807: 121 pgs: 121 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T10:03:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:41 smithi033 ceph-mon[96568]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T10:03:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:41 smithi033 ceph-mon[98546]: pgmap v807: 121 pgs: 121 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T10:03:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:41 smithi033 ceph-mon[98546]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T10:03:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:03:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:03:41] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.51.0" 2024-09-21T10:03:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:41 smithi105 ceph-mon[86649]: pgmap v807: 121 pgs: 121 active+clean; 154 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-09-21T10:03:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:41 smithi105 ceph-mon[86649]: osdmap e782: 8 total, 8 up, 8 in 2024-09-21T10:03:42.664 INFO:tasks.rados.rados.0.smithi033.stdout:3889: done (5 left) 2024-09-21T10:03:42.664 INFO:tasks.rados.rados.0.smithi033.stdout:3890: snap_remove snap 346 2024-09-21T10:03:43.585 INFO:tasks.rados.rados.0.smithi033.stdout:3890: done (5 left) 2024-09-21T10:03:43.585 INFO:tasks.rados.rados.0.smithi033.stdout:3891: copy_from oid 14 from oid 1 current snap is 351 2024-09-21T10:03:43.586 INFO:tasks.rados.rados.0.smithi033.stdout:3892: read oid 34 snap -1 2024-09-21T10:03:43.586 INFO:tasks.rados.rados.0.smithi033.stdout:3892: expect (ObjNum 1209 snap 349 seq_num 1209) 2024-09-21T10:03:43.586 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 29 current snap is 351 2024-09-21T10:03:43.586 INFO:tasks.rados.rados.0.smithi033.stdout:3893: seq_num 1228 ranges {7053312=573440,7626752=761856,8388608=40960} 2024-09-21T10:03:43.628 INFO:tasks.rados.rados.0.smithi033.stdout:3893: writing smithi03385152-29 from 7053312 to 7626752 tid 1 2024-09-21T10:03:43.632 INFO:tasks.rados.rados.0.smithi033.stdout:3893: writing smithi03385152-29 from 7626752 to 8388608 tid 2 2024-09-21T10:03:43.633 INFO:tasks.rados.rados.0.smithi033.stdout:3893: writing smithi03385152-29 from 8388608 to 8429568 tid 3 2024-09-21T10:03:43.633 INFO:tasks.rados.rados.0.smithi033.stdout:3894: read oid 38 snap -1 2024-09-21T10:03:43.634 INFO:tasks.rados.rados.0.smithi033.stdout:3894: expect (ObjNum 1202 snap 348 seq_num 1202) 2024-09-21T10:03:43.634 INFO:tasks.rados.rados.0.smithi033.stdout:3895: rmattr oid 26 current snap is 351 2024-09-21T10:03:43.634 INFO:tasks.rados.rados.0.smithi033.stdout:3895: done (9 left) 2024-09-21T10:03:43.634 INFO:tasks.rados.rados.0.smithi033.stdout:3896: read oid 25 snap -1 2024-09-21T10:03:43.634 INFO:tasks.rados.rados.0.smithi033.stdout:3896: expect (ObjNum 1225 snap 351 seq_num 1225) 2024-09-21T10:03:43.634 INFO:tasks.rados.rados.0.smithi033.stdout:3897: snap_remove snap 350 2024-09-21T10:03:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:43 smithi033 ceph-mon[98546]: pgmap v809: 121 pgs: 121 active+clean; 178 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 6.6 MiB/s wr, 26 op/s 2024-09-21T10:03:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:43 smithi033 ceph-mon[98546]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T10:03:43.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:43 smithi033 ceph-mon[98546]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T10:03:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:43 smithi033 ceph-mon[96568]: pgmap v809: 121 pgs: 121 active+clean; 178 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 6.6 MiB/s wr, 26 op/s 2024-09-21T10:03:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:43 smithi033 ceph-mon[96568]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T10:03:43.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:43 smithi033 ceph-mon[96568]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T10:03:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:43 smithi105 ceph-mon[86649]: pgmap v809: 121 pgs: 121 active+clean; 178 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 25 MiB/s rd, 6.6 MiB/s wr, 26 op/s 2024-09-21T10:03:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:43 smithi105 ceph-mon[86649]: osdmap e783: 8 total, 8 up, 8 in 2024-09-21T10:03:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:43 smithi105 ceph-mon[86649]: osdmap e784: 8 total, 8 up, 8 in 2024-09-21T10:03:44.594 INFO:tasks.rados.rados.0.smithi033.stdout:3897: done (10 left) 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3898: read oid 3 snap -1 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3898: expect (ObjNum 1192 snap 345 seq_num 1192) 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3899: snap_create 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3884: finishing write tid 4 to smithi03385152-21 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3884: finishing write tid 5 to smithi03385152-21 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 2473 (ObjNum 1226 snap 351 seq_num 1226) dirty exists 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3884: left oid 21 (ObjNum 1226 snap 351 seq_num 1226) 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3886: finishing write tid 1 to smithi03385152-27 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3887: finishing copy_from racing read to smithi03385152-47 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3886: finishing write tid 2 to smithi03385152-27 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3886: finishing write tid 3 to smithi03385152-27 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3886: finishing write tid 4 to smithi03385152-27 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:3886: finishing write tid 5 to smithi03385152-27 2024-09-21T10:03:44.595 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2508 (ObjNum 1227 snap 351 seq_num 1227) dirty exists 2024-09-21T10:03:44.596 INFO:tasks.rados.rados.0.smithi033.stdout:3886: left oid 27 (ObjNum 1227 snap 351 seq_num 1227) 2024-09-21T10:03:44.596 INFO:tasks.rados.rados.0.smithi033.stdout:3887: finishing copy_from to smithi03385152-47 2024-09-21T10:03:44.596 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 47 v 2491 (ObjNum 1146 snap 329 seq_num 1146) dirty exists 2024-09-21T10:03:44.596 INFO:tasks.rados.rados.0.smithi033.stdout:3885: expect (ObjNum 1181 snap 338 seq_num 1181) 2024-09-21T10:03:45.126 INFO:tasks.rados.rados.0.smithi033.stdout:3888: expect (ObjNum 1121 snap 324 seq_num 1121) 2024-09-21T10:03:45.177 INFO:tasks.rados.rados.0.smithi033.stdout:3891: finishing copy_from to smithi03385152-14 2024-09-21T10:03:45.177 INFO:tasks.rados.rados.0.smithi033.stdout:3891: got expected ENOENT (src dne) 2024-09-21T10:03:45.177 INFO:tasks.rados.rados.0.smithi033.stdout:3891: finishing copy_from racing read to smithi03385152-14 2024-09-21T10:03:45.177 INFO:tasks.rados.rados.0.smithi033.stdout:3892: expect (ObjNum 1209 snap 349 seq_num 1209) 2024-09-21T10:03:45.239 INFO:tasks.rados.rados.0.smithi033.stdout:3893: finishing write tid 1 to smithi03385152-29 2024-09-21T10:03:45.239 INFO:tasks.rados.rados.0.smithi033.stdout:3893: finishing write tid 2 to smithi03385152-29 2024-09-21T10:03:45.239 INFO:tasks.rados.rados.0.smithi033.stdout:3896: expect (ObjNum 1225 snap 351 seq_num 1225) 2024-09-21T10:03:45.273 INFO:tasks.rados.rados.0.smithi033.stdout:3893: finishing write tid 3 to smithi03385152-29 2024-09-21T10:03:45.274 INFO:tasks.rados.rados.0.smithi033.stdout:3893: finishing write tid 4 to smithi03385152-29 2024-09-21T10:03:45.274 INFO:tasks.rados.rados.0.smithi033.stdout:3893: finishing write tid 5 to smithi03385152-29 2024-09-21T10:03:45.274 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 29 v 2786 (ObjNum 1228 snap 351 seq_num 1228) dirty exists 2024-09-21T10:03:45.274 INFO:tasks.rados.rados.0.smithi033.stdout:3893: left oid 29 (ObjNum 1228 snap 351 seq_num 1228) 2024-09-21T10:03:45.274 INFO:tasks.rados.rados.0.smithi033.stdout:3894: expect (ObjNum 1202 snap 348 seq_num 1202) 2024-09-21T10:03:45.664 INFO:tasks.rados.rados.0.smithi033.stdout:3898: expect (ObjNum 1192 snap 345 seq_num 1192) 2024-09-21T10:03:45.757 INFO:tasks.rados.rados.0.smithi033.stdout:3884: done (11 left) 2024-09-21T10:03:45.757 INFO:tasks.rados.rados.0.smithi033.stdout:3885: done (10 left) 2024-09-21T10:03:45.757 INFO:tasks.rados.rados.0.smithi033.stdout:3886: done (9 left) 2024-09-21T10:03:45.757 INFO:tasks.rados.rados.0.smithi033.stdout:3887: done (8 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3888: done (7 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3891: done (6 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3892: done (5 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3893: done (4 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3894: done (3 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3896: done (2 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3898: done (1 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3899: done (0 left) 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 352 2024-09-21T10:03:45.758 INFO:tasks.rados.rados.0.smithi033.stdout:3900: seq_num 1229 ranges {0=589824,589824=647168,1236992=155648} 2024-09-21T10:03:45.760 INFO:tasks.rados.rados.0.smithi033.stdout:3900: writing smithi03385152-1 from 0 to 589824 tid 1 2024-09-21T10:03:45.763 INFO:tasks.rados.rados.0.smithi033.stdout:3900: writing smithi03385152-1 from 589824 to 1236992 tid 2 2024-09-21T10:03:45.764 INFO:tasks.rados.rados.0.smithi033.stdout:3900: writing smithi03385152-1 from 1236992 to 1392640 tid 3 2024-09-21T10:03:45.764 INFO:tasks.rados.rados.0.smithi033.stdout:3901: delete oid 50 current snap is 352 2024-09-21T10:03:45.768 INFO:tasks.rados.rados.0.smithi033.stdout:3901: done (1 left) 2024-09-21T10:03:45.768 INFO:tasks.rados.rados.0.smithi033.stdout:3902: snap_create 2024-09-21T10:03:45.768 INFO:tasks.rados.rados.0.smithi033.stdout:3900: finishing write tid 1 to smithi03385152-1 2024-09-21T10:03:45.769 INFO:tasks.rados.rados.0.smithi033.stdout:3900: finishing write tid 2 to smithi03385152-1 2024-09-21T10:03:45.769 INFO:tasks.rados.rados.0.smithi033.stdout:3900: finishing write tid 3 to smithi03385152-1 2024-09-21T10:03:45.770 INFO:tasks.rados.rados.0.smithi033.stdout:3900: finishing write tid 4 to smithi03385152-1 2024-09-21T10:03:45.771 INFO:tasks.rados.rados.0.smithi033.stdout:3900: finishing write tid 5 to smithi03385152-1 2024-09-21T10:03:45.771 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 2479 (ObjNum 1229 snap 352 seq_num 1229) dirty exists 2024-09-21T10:03:45.771 INFO:tasks.rados.rados.0.smithi033.stdout:3900: left oid 1 (ObjNum 1229 snap 352 seq_num 1229) 2024-09-21T10:03:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:45 smithi033 ceph-mon[98546]: pgmap v812: 121 pgs: 121 active+clean; 192 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 8.2 MiB/s wr, 26 op/s 2024-09-21T10:03:45.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:45 smithi033 ceph-mon[98546]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T10:03:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:45 smithi033 ceph-mon[96568]: pgmap v812: 121 pgs: 121 active+clean; 192 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 8.2 MiB/s wr, 26 op/s 2024-09-21T10:03:45.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:45 smithi033 ceph-mon[96568]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T10:03:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:45 smithi105 ceph-mon[86649]: pgmap v812: 121 pgs: 121 active+clean; 192 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 8.2 MiB/s wr, 26 op/s 2024-09-21T10:03:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:45 smithi105 ceph-mon[86649]: osdmap e785: 8 total, 8 up, 8 in 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:3900: done (1 left) 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:3902: done (0 left) 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:3903: rmattr oid 15 current snap is 353 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:3903: done (0 left) 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:3904: read oid 37 snap -1 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:3904: expect deleted 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:3905: read oid 2 snap -1 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:3905: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:03:46.603 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 353 2024-09-21T10:03:46.604 INFO:tasks.rados.rados.0.smithi033.stdout:3906: seq_num 1231 ranges {2408448=622592,3031040=417792,3448832=385024} 2024-09-21T10:03:46.619 INFO:tasks.rados.rados.0.smithi033.stdout:3906: writing smithi03385152-6 from 2408448 to 3031040 tid 1 2024-09-21T10:03:46.622 INFO:tasks.rados.rados.0.smithi033.stdout:3906: writing smithi03385152-6 from 3031040 to 3448832 tid 2 2024-09-21T10:03:46.625 INFO:tasks.rados.rados.0.smithi033.stdout:3906: writing smithi03385152-6 from 3448832 to 3833856 tid 3 2024-09-21T10:03:46.625 INFO:tasks.rados.rados.0.smithi033.stdout:3907: copy_from oid 38 from oid 19 current snap is 353 2024-09-21T10:03:46.625 INFO:tasks.rados.rados.0.smithi033.stdout:3905: expect (ObjNum 0 snap 0 seq_num 0) 2024-09-21T10:03:46.628 INFO:tasks.rados.rados.0.smithi033.stdout:3906: finishing write tid 1 to smithi03385152-6 2024-09-21T10:03:46.628 INFO:tasks.rados.rados.0.smithi033.stdout:3906: finishing write tid 2 to smithi03385152-6 2024-09-21T10:03:46.628 INFO:tasks.rados.rados.0.smithi033.stdout:3904: done (3 left) 2024-09-21T10:03:46.628 INFO:tasks.rados.rados.0.smithi033.stdout:3905: done (2 left) 2024-09-21T10:03:46.628 INFO:tasks.rados.rados.0.smithi033.stdout:3908: snap_create 2024-09-21T10:03:46.628 INFO:tasks.rados.rados.0.smithi033.stdout:3906: finishing write tid 3 to smithi03385152-6 2024-09-21T10:03:46.628 INFO:tasks.rados.rados.0.smithi033.stdout:3906: finishing write tid 4 to smithi03385152-6 2024-09-21T10:03:46.629 INFO:tasks.rados.rados.0.smithi033.stdout:3906: finishing write tid 5 to smithi03385152-6 2024-09-21T10:03:46.629 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 2512 (ObjNum 1231 snap 353 seq_num 1231) dirty exists 2024-09-21T10:03:46.629 INFO:tasks.rados.rados.0.smithi033.stdout:3906: left oid 6 (ObjNum 1231 snap 353 seq_num 1231) 2024-09-21T10:03:46.644 INFO:tasks.rados.rados.0.smithi033.stdout:3907: finishing copy_from racing read to smithi03385152-38 2024-09-21T10:03:46.649 INFO:tasks.rados.rados.0.smithi033.stdout:3907: finishing copy_from to smithi03385152-38 2024-09-21T10:03:46.649 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2489 (ObjNum 1127 snap 325 seq_num 1127) dirty exists 2024-09-21T10:03:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:46 smithi033 ceph-mon[96568]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T10:03:46.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:46 smithi033 ceph-mon[98546]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T10:03:46.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:46 smithi105 ceph-mon[86649]: osdmap e786: 8 total, 8 up, 8 in 2024-09-21T10:03:47.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:03:47.609 INFO:tasks.rados.rados.0.smithi033.stdout:3906: done (2 left) 2024-09-21T10:03:47.609 INFO:tasks.rados.rados.0.smithi033.stdout:3907: done (1 left) 2024-09-21T10:03:47.609 INFO:tasks.rados.rados.0.smithi033.stdout:3908: done (0 left) 2024-09-21T10:03:47.609 INFO:tasks.rados.rados.0.smithi033.stdout:3909: delete oid 26 current snap is 354 2024-09-21T10:03:47.613 INFO:tasks.rados.rados.0.smithi033.stdout:3909: done (0 left) 2024-09-21T10:03:47.614 INFO:tasks.rados.rados.0.smithi033.stdout:3910: snap_remove snap 351 2024-09-21T10:03:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:47 smithi033 ceph-mon[98546]: pgmap v815: 121 pgs: 121 active+clean; 195 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 5.3 MiB/s wr, 37 op/s 2024-09-21T10:03:47.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:47 smithi033 ceph-mon[98546]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T10:03:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:47 smithi033 ceph-mon[96568]: pgmap v815: 121 pgs: 121 active+clean; 195 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 5.3 MiB/s wr, 37 op/s 2024-09-21T10:03:47.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:47 smithi033 ceph-mon[96568]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T10:03:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:47 smithi105 ceph-mon[86649]: pgmap v815: 121 pgs: 121 active+clean; 195 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 23 MiB/s rd, 5.3 MiB/s wr, 37 op/s 2024-09-21T10:03:48.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:47 smithi105 ceph-mon[86649]: osdmap e787: 8 total, 8 up, 8 in 2024-09-21T10:03:48.587 INFO:tasks.rados.rados.0.smithi033.stdout:3910: done (0 left) 2024-09-21T10:03:48.587 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 1 current snap is 354 2024-09-21T10:03:48.587 INFO:tasks.rados.rados.0.smithi033.stdout:3911: seq_num 1233 ranges {1392640=655360,2048000=638976,2686976=163840} 2024-09-21T10:03:48.598 INFO:tasks.rados.rados.0.smithi033.stdout:3911: writing smithi03385152-1 from 1392640 to 2048000 tid 1 2024-09-21T10:03:48.601 INFO:tasks.rados.rados.0.smithi033.stdout:3911: writing smithi03385152-1 from 2048000 to 2686976 tid 2 2024-09-21T10:03:48.603 INFO:tasks.rados.rados.0.smithi033.stdout:3911: writing smithi03385152-1 from 2686976 to 2850816 tid 3 2024-09-21T10:03:48.603 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 354 2024-09-21T10:03:48.603 INFO:tasks.rados.rados.0.smithi033.stdout:3912: seq_num 1234 ranges {13590528=671744,14262272=516096,14778368=286720} 2024-09-21T10:03:48.681 INFO:tasks.rados.rados.0.smithi033.stdout:3912: writing smithi03385152-31 from 13590528 to 14262272 tid 1 2024-09-21T10:03:48.684 INFO:tasks.rados.rados.0.smithi033.stdout:3912: writing smithi03385152-31 from 14262272 to 14778368 tid 2 2024-09-21T10:03:48.686 INFO:tasks.rados.rados.0.smithi033.stdout:3912: writing smithi03385152-31 from 14778368 to 15065088 tid 3 2024-09-21T10:03:48.686 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 38 current snap is 354 2024-09-21T10:03:48.686 INFO:tasks.rados.rados.0.smithi033.stdout:3911: finishing write tid 1 to smithi03385152-1 2024-09-21T10:03:48.686 INFO:tasks.rados.rados.0.smithi033.stdout:3913: seq_num 1235 ranges {4718592=688128,5406720=802816} 2024-09-21T10:03:48.715 INFO:tasks.rados.rados.0.smithi033.stdout:3913: writing smithi03385152-38 from 4718592 to 5406720 tid 1 2024-09-21T10:03:48.720 INFO:tasks.rados.rados.0.smithi033.stdout:3913: writing smithi03385152-38 from 5406720 to 6209536 tid 2 2024-09-21T10:03:48.720 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 44 current snap is 354 2024-09-21T10:03:48.721 INFO:tasks.rados.rados.0.smithi033.stdout:3914: seq_num 1236 ranges {1982464=704512,2686976=688128,3375104=114688} 2024-09-21T10:03:48.735 INFO:tasks.rados.rados.0.smithi033.stdout:3914: writing smithi03385152-44 from 1982464 to 2686976 tid 1 2024-09-21T10:03:48.739 INFO:tasks.rados.rados.0.smithi033.stdout:3914: writing smithi03385152-44 from 2686976 to 3375104 tid 2 2024-09-21T10:03:48.740 INFO:tasks.rados.rados.0.smithi033.stdout:3914: writing smithi03385152-44 from 3375104 to 3489792 tid 3 2024-09-21T10:03:48.740 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 354 2024-09-21T10:03:48.740 INFO:tasks.rados.rados.0.smithi033.stdout:3911: finishing write tid 2 to smithi03385152-1 2024-09-21T10:03:48.740 INFO:tasks.rados.rados.0.smithi033.stdout:3911: finishing write tid 3 to smithi03385152-1 2024-09-21T10:03:48.740 INFO:tasks.rados.rados.0.smithi033.stdout:3911: finishing write tid 4 to smithi03385152-1 2024-09-21T10:03:48.740 INFO:tasks.rados.rados.0.smithi033.stdout:3911: finishing write tid 5 to smithi03385152-1 2024-09-21T10:03:48.740 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 1 v 2483 (ObjNum 1233 snap 354 seq_num 1233) dirty exists 2024-09-21T10:03:48.740 INFO:tasks.rados.rados.0.smithi033.stdout:3911: left oid 1 (ObjNum 1233 snap 354 seq_num 1233) 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3912: finishing write tid 1 to smithi03385152-31 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3912: finishing write tid 2 to smithi03385152-31 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3912: finishing write tid 3 to smithi03385152-31 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3912: finishing write tid 4 to smithi03385152-31 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3912: finishing write tid 5 to smithi03385152-31 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2797 (ObjNum 1234 snap 354 seq_num 1234) dirty exists 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3912: left oid 31 (ObjNum 1234 snap 354 seq_num 1234) 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3913: finishing write tid 1 to smithi03385152-38 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3913: finishing write tid 2 to smithi03385152-38 2024-09-21T10:03:48.741 INFO:tasks.rados.rados.0.smithi033.stdout:3915: seq_num 1237 ranges {3596288=720896,4317184=573440,4890624=229376} 2024-09-21T10:03:48.763 INFO:tasks.rados.rados.0.smithi033.stdout:3915: writing smithi03385152-5 from 3596288 to 4317184 tid 1 2024-09-21T10:03:48.766 INFO:tasks.rados.rados.0.smithi033.stdout:3915: writing smithi03385152-5 from 4317184 to 4890624 tid 2 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3915: writing smithi03385152-5 from 4890624 to 5120000 tid 3 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3911: done (4 left) 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3912: done (3 left) 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3916: rmattr oid 19 current snap is 354 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3913: finishing write tid 3 to smithi03385152-38 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3913: finishing write tid 4 to smithi03385152-38 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 38 v 2497 (ObjNum 1235 snap 354 seq_num 1235) dirty exists 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3913: left oid 38 (ObjNum 1235 snap 354 seq_num 1235) 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3914: finishing write tid 1 to smithi03385152-44 2024-09-21T10:03:48.769 INFO:tasks.rados.rados.0.smithi033.stdout:3913: done (3 left) 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3917: rollback oid 28 current snap is 354 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 28 to 352 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3914: finishing write tid 2 to smithi03385152-44 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3914: finishing write tid 3 to smithi03385152-44 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3914: finishing write tid 4 to smithi03385152-44 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3914: finishing write tid 5 to smithi03385152-44 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 44 v 2643 (ObjNum 1236 snap 354 seq_num 1236) dirty exists 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3914: left oid 44 (ObjNum 1236 snap 354 seq_num 1236) 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3914: done (3 left) 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3918: rmattr oid 20 current snap is 354 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3918: done (3 left) 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3919: rmattr oid 41 current snap is 354 2024-09-21T10:03:48.770 INFO:tasks.rados.rados.0.smithi033.stdout:3919: done (3 left) 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3920: read oid 24 snap -1 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3920: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3921: rollback oid 36 current snap is 354 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 36 to 348 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3922: read oid 32 snap -1 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3922: expect (ObjNum 1182 snap 339 seq_num 1182) 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3923: setattr oid 27 current snap is 354 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3915: finishing write tid 1 to smithi03385152-5 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3924: rmattr oid 41 current snap is 354 2024-09-21T10:03:48.771 INFO:tasks.rados.rados.0.smithi033.stdout:3924: done (7 left) 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:3925: read oid 21 snap -1 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:3925: expect (ObjNum 1226 snap 351 seq_num 1226) 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:3926: read oid 26 snap -1 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:3926: expect deleted 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:3927: copy_from oid 33 from oid 49 current snap is 354 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:3928: rollback oid 17 current snap is 354 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 17 to 353 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:3929: rollback oid 12 current snap is 354 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 12 to 352 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 354 2024-09-21T10:03:48.772 INFO:tasks.rados.rados.0.smithi033.stdout:3930: seq_num 1238 ranges {3833856=737280,4571136=458752,5029888=344064} 2024-09-21T10:03:48.796 INFO:tasks.rados.rados.0.smithi033.stdout:3930: writing smithi03385152-6 from 3833856 to 4571136 tid 1 2024-09-21T10:03:48.799 INFO:tasks.rados.rados.0.smithi033.stdout:3930: writing smithi03385152-6 from 4571136 to 5029888 tid 2 2024-09-21T10:03:48.801 INFO:tasks.rados.rados.0.smithi033.stdout:3930: writing smithi03385152-6 from 5029888 to 5373952 tid 3 2024-09-21T10:03:48.801 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 9 current snap is 354 2024-09-21T10:03:48.801 INFO:tasks.rados.rados.0.smithi033.stdout:3915: finishing write tid 2 to smithi03385152-5 2024-09-21T10:03:48.801 INFO:tasks.rados.rados.0.smithi033.stdout:3931: seq_num 1239 ranges {3227648=753664,3981312=745472,4726784=57344} 2024-09-21T10:03:48.823 INFO:tasks.rados.rados.0.smithi033.stdout:3931: writing smithi03385152-9 from 3227648 to 3981312 tid 1 2024-09-21T10:03:48.827 INFO:tasks.rados.rados.0.smithi033.stdout:3931: writing smithi03385152-9 from 3981312 to 4726784 tid 2 2024-09-21T10:03:48.828 INFO:tasks.rados.rados.0.smithi033.stdout:3931: writing smithi03385152-9 from 4726784 to 4784128 tid 3 2024-09-21T10:03:48.828 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 354 2024-09-21T10:03:48.828 INFO:tasks.rados.rados.0.smithi033.stdout:3932: seq_num 1240 ranges {9641984=778240,10420224=671744,11091968=131072} 2024-09-21T10:03:48.885 INFO:tasks.rados.rados.0.smithi033.stdout:3932: writing smithi03385152-42 from 9641984 to 10420224 tid 1 2024-09-21T10:03:48.889 INFO:tasks.rados.rados.0.smithi033.stdout:3932: writing smithi03385152-42 from 10420224 to 11091968 tid 2 2024-09-21T10:03:48.889 INFO:tasks.rados.rados.0.smithi033.stdout:3932: writing smithi03385152-42 from 11091968 to 11223040 tid 3 2024-09-21T10:03:48.890 INFO:tasks.rados.rados.0.smithi033.stdout:3933: delete oid 47 current snap is 354 2024-09-21T10:03:48.890 INFO:tasks.rados.rados.0.smithi033.stdout:3917: finishing rollback tid 0 to smithi03385152-28 2024-09-21T10:03:48.890 INFO:tasks.rados.rados.0.smithi033.stdout:3915: finishing write tid 3 to smithi03385152-5 2024-09-21T10:03:48.890 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2495 (ObjNum 1127 snap 325 seq_num 1127) dirty exists 2024-09-21T10:03:48.890 INFO:tasks.rados.rados.0.smithi033.stdout:3922: expect (ObjNum 1182 snap 339 seq_num 1182) 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:3917: finishing rollback tid 1 to smithi03385152-28 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:3917: finishing rollback tid 2 to smithi03385152-28 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 28 v 1844 (ObjNum 1214 snap 351 seq_num 1214) dirty exists 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:3921: finishing rollback tid 0 to smithi03385152-36 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:3915: finishing write tid 4 to smithi03385152-5 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:3915: finishing write tid 5 to smithi03385152-5 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2431 (ObjNum 1237 snap 354 seq_num 1237) dirty exists 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:3915: left oid 5 (ObjNum 1237 snap 354 seq_num 1237) 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2513 (ObjNum 1227 snap 351 seq_num 1227) dirty exists 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:3921: finishing rollback tid 1 to smithi03385152-36 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2800 (ObjNum 33 snap 0 seq_num 91456864) dirty dne 2024-09-21T10:03:48.908 INFO:tasks.rados.rados.0.smithi033.stdout:3920: expect (ObjNum 1103 snap 324 seq_num 1103) 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:3927: finishing copy_from racing read to smithi03385152-33 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:3928: finishing rollback tid 0 to smithi03385152-17 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:3928: finishing rollback tid 1 to smithi03385152-17 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:3928: finishing rollback tid 2 to smithi03385152-17 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 17 v 2502 (ObjNum 1192 snap 345 seq_num 1192) dirty exists 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:3929: finishing rollback tid 0 to smithi03385152-12 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:3929: finishing rollback tid 1 to smithi03385152-12 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:3929: finishing rollback tid 2 to smithi03385152-12 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2505 (ObjNum 1181 snap 338 seq_num 1181) dirty exists 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:3927: finishing copy_from to smithi03385152-33 2024-09-21T10:03:48.952 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 2490 (ObjNum 1170 snap 335 seq_num 1170) dirty exists 2024-09-21T10:03:48.953 INFO:tasks.rados.rados.0.smithi033.stdout:3925: expect (ObjNum 1226 snap 351 seq_num 1226) 2024-09-21T10:03:48.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:48 smithi033 ceph-mon[98546]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T10:03:48.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:48 smithi033 ceph-mon[98546]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T10:03:48.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:48 smithi033 ceph-mon[96568]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T10:03:48.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:48 smithi033 ceph-mon[96568]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T10:03:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:48 smithi105 ceph-mon[86649]: osdmap e788: 8 total, 8 up, 8 in 2024-09-21T10:03:49.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:48 smithi105 ceph-mon[86649]: osdmap e789: 8 total, 8 up, 8 in 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:3930: finishing write tid 1 to smithi03385152-6 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:3930: finishing write tid 2 to smithi03385152-6 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:3930: finishing write tid 3 to smithi03385152-6 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:3930: finishing write tid 4 to smithi03385152-6 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:3930: finishing write tid 5 to smithi03385152-6 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 2517 (ObjNum 1238 snap 354 seq_num 1238) dirty exists 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:3930: left oid 6 (ObjNum 1238 snap 354 seq_num 1238) 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:3931: finishing write tid 1 to smithi03385152-9 2024-09-21T10:03:49.138 INFO:tasks.rados.rados.0.smithi033.stdout:3931: finishing write tid 2 to smithi03385152-9 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3931: finishing write tid 3 to smithi03385152-9 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3931: finishing write tid 4 to smithi03385152-9 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3931: finishing write tid 5 to smithi03385152-9 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 2501 (ObjNum 1239 snap 354 seq_num 1239) dirty exists 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3931: left oid 9 (ObjNum 1239 snap 354 seq_num 1239) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3932: finishing write tid 1 to smithi03385152-42 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3915: done (15 left) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3916: done (14 left) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3917: done (13 left) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3920: done (12 left) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3921: done (11 left) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3922: done (10 left) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3923: done (9 left) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3925: done (8 left) 2024-09-21T10:03:49.139 INFO:tasks.rados.rados.0.smithi033.stdout:3926: done (7 left) 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3927: done (6 left) 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3928: done (5 left) 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3929: done (4 left) 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3930: done (3 left) 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3931: done (2 left) 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3933: done (1 left) 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3934: read oid 7 snap -1 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3934: expect deleted 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3935: copy_from oid 37 from oid 40 current snap is 354 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3932: finishing write tid 2 to smithi03385152-42 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3932: finishing write tid 3 to smithi03385152-42 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3932: finishing write tid 4 to smithi03385152-42 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3932: finishing write tid 5 to smithi03385152-42 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 2505 (ObjNum 1240 snap 354 seq_num 1240) dirty exists 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3932: left oid 42 (ObjNum 1240 snap 354 seq_num 1240) 2024-09-21T10:03:49.140 INFO:tasks.rados.rados.0.smithi033.stdout:3932: done (2 left) 2024-09-21T10:03:49.141 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 6 current snap is 354 2024-09-21T10:03:49.141 INFO:tasks.rados.rados.0.smithi033.stdout:3936: seq_num 1242 ranges {5373952=409600,5783552=442368,6225920=655360,6881280=106496} 2024-09-21T10:03:49.169 INFO:tasks.rados.rados.0.smithi033.stdout:3936: writing smithi03385152-6 from 5373952 to 5783552 tid 1 2024-09-21T10:03:49.171 INFO:tasks.rados.rados.0.smithi033.stdout:3936: writing smithi03385152-6 from 5783552 to 6225920 tid 2 2024-09-21T10:03:49.175 INFO:tasks.rados.rados.0.smithi033.stdout:3936: writing smithi03385152-6 from 6225920 to 6881280 tid 3 2024-09-21T10:03:49.176 INFO:tasks.rados.rados.0.smithi033.stdout:3936: writing smithi03385152-6 from 6881280 to 6987776 tid 4 2024-09-21T10:03:49.176 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 30 current snap is 354 2024-09-21T10:03:49.176 INFO:tasks.rados.rados.0.smithi033.stdout:3937: seq_num 1243 ranges {3227648=425984,3653632=729088,4382720=466944,4849664=8192} 2024-09-21T10:03:49.196 INFO:tasks.rados.rados.0.smithi033.stdout:3937: writing smithi03385152-30 from 3227648 to 3653632 tid 1 2024-09-21T10:03:49.200 INFO:tasks.rados.rados.0.smithi033.stdout:3937: writing smithi03385152-30 from 3653632 to 4382720 tid 2 2024-09-21T10:03:49.203 INFO:tasks.rados.rados.0.smithi033.stdout:3937: writing smithi03385152-30 from 4382720 to 4849664 tid 3 2024-09-21T10:03:49.203 INFO:tasks.rados.rados.0.smithi033.stdout:3937: writing smithi03385152-30 from 4849664 to 4857856 tid 4 2024-09-21T10:03:49.203 INFO:tasks.rados.rados.0.smithi033.stdout:3938: snap_remove snap 352 2024-09-21T10:03:49.618 INFO:tasks.rados.rados.0.smithi033.stdout:3935: finishing copy_from racing read to smithi03385152-37 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3935: finishing copy_from to smithi03385152-37 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 37 v 2493 (ObjNum 1216 snap 351 seq_num 1216) dirty exists 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3936: finishing write tid 1 to smithi03385152-6 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3936: finishing write tid 2 to smithi03385152-6 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3936: finishing write tid 3 to smithi03385152-6 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3936: finishing write tid 4 to smithi03385152-6 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3936: finishing write tid 5 to smithi03385152-6 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3936: finishing write tid 6 to smithi03385152-6 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 2522 (ObjNum 1242 snap 354 seq_num 1242) dirty exists 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3936: left oid 6 (ObjNum 1242 snap 354 seq_num 1242) 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3937: finishing write tid 1 to smithi03385152-30 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3937: finishing write tid 2 to smithi03385152-30 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3937: finishing write tid 3 to smithi03385152-30 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3937: finishing write tid 4 to smithi03385152-30 2024-09-21T10:03:49.619 INFO:tasks.rados.rados.0.smithi033.stdout:3937: finishing write tid 5 to smithi03385152-30 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:3937: finishing write tid 6 to smithi03385152-30 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 30 v 2806 (ObjNum 1243 snap 354 seq_num 1243) dirty exists 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:3937: left oid 30 (ObjNum 1243 snap 354 seq_num 1243) 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:3934: done (4 left) 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:3935: done (3 left) 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:3936: done (2 left) 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:3937: done (1 left) 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:3938: done (0 left) 2024-09-21T10:03:49.620 INFO:tasks.rados.rados.0.smithi033.stdout:3939: snap_remove snap 353 2024-09-21T10:03:49.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:49 smithi033 ceph-mon[98546]: pgmap v818: 121 pgs: 121 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T10:03:49.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:49 smithi033 ceph-mon[96568]: pgmap v818: 121 pgs: 121 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T10:03:50.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:49 smithi105 ceph-mon[86649]: pgmap v818: 121 pgs: 121 active+clean; 196 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 1.9 MiB/s wr, 20 op/s 2024-09-21T10:03:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:03:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:03:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:03:50.636 INFO:tasks.rados.rados.0.smithi033.stdout:3939: done (0 left) 2024-09-21T10:03:50.636 INFO:tasks.rados.rados.0.smithi033.stdout:3940: rollback oid 33 current snap is 354 2024-09-21T10:03:50.636 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 33 to 348 2024-09-21T10:03:50.636 INFO:tasks.rados.rados.0.smithi033.stdout:3941: rollback oid 7 current snap is 354 2024-09-21T10:03:50.636 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 7 to 348 2024-09-21T10:03:50.636 INFO:tasks.rados.rados.0.smithi033.stdout:3942: read oid 22 snap -1 2024-09-21T10:03:50.637 INFO:tasks.rados.rados.0.smithi033.stdout:3942: expect (ObjNum 1213 snap 350 seq_num 1213) 2024-09-21T10:03:50.637 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 31 current snap is 354 2024-09-21T10:03:50.637 INFO:tasks.rados.rados.0.smithi033.stdout:3943: seq_num 1244 ranges {15065088=442368,15507456=614400,16121856=589824} 2024-09-21T10:03:50.722 INFO:tasks.rados.rados.0.smithi033.stdout:3943: writing smithi03385152-31 from 15065088 to 15507456 tid 1 2024-09-21T10:03:50.726 INFO:tasks.rados.rados.0.smithi033.stdout:3943: writing smithi03385152-31 from 15507456 to 16121856 tid 2 2024-09-21T10:03:50.731 INFO:tasks.rados.rados.0.smithi033.stdout:3943: writing smithi03385152-31 from 16121856 to 16711680 tid 3 2024-09-21T10:03:50.731 INFO:tasks.rados.rados.0.smithi033.stdout:3944: copy_from oid 9 from oid 48 current snap is 354 2024-09-21T10:03:50.731 INFO:tasks.rados.rados.0.smithi033.stdout:3941: finishing rollback tid 1 to smithi03385152-7 2024-09-21T10:03:50.731 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 7 v 0 (ObjNum 177 snap 0 seq_num 91278400) dirty dne 2024-09-21T10:03:50.731 INFO:tasks.rados.rados.0.smithi033.stdout:3940: finishing rollback tid 0 to smithi03385152-33 2024-09-21T10:03:50.731 INFO:tasks.rados.rados.0.smithi033.stdout:3940: finishing rollback tid 1 to smithi03385152-33 2024-09-21T10:03:50.731 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 33 v 2524 (ObjNum 33 snap 0 seq_num 72673360) dirty dne 2024-09-21T10:03:50.731 INFO:tasks.rados.rados.0.smithi033.stdout:3942: expect (ObjNum 1213 snap 350 seq_num 1213) 2024-09-21T10:03:50.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:50 smithi033 ceph-mon[98546]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T10:03:50.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:50 smithi033 ceph-mon[96568]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T10:03:51.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:50 smithi105 ceph-mon[86649]: osdmap e790: 8 total, 8 up, 8 in 2024-09-21T10:03:51.585 INFO:tasks.rados.rados.0.smithi033.stdout:3943: finishing write tid 1 to smithi03385152-31 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3943: finishing write tid 2 to smithi03385152-31 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3943: finishing write tid 3 to smithi03385152-31 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3940: done (4 left) 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3941: done (3 left) 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3942: done (2 left) 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3945: read oid 29 snap 348 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3945: expect (ObjNum 1191 snap 344 seq_num 1191) 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3946: delete oid 38 current snap is 354 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3943: finishing write tid 4 to smithi03385152-31 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3943: finishing write tid 5 to smithi03385152-31 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 31 v 2822 (ObjNum 1244 snap 354 seq_num 1244) dirty exists 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3943: left oid 31 (ObjNum 1244 snap 354 seq_num 1244) 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3944: finishing copy_from racing read to smithi03385152-9 2024-09-21T10:03:51.586 INFO:tasks.rados.rados.0.smithi033.stdout:3944: finishing copy_from to smithi03385152-9 2024-09-21T10:03:51.587 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 2434 (ObjNum 1116 snap 324 seq_num 1116) dirty exists 2024-09-21T10:03:51.588 INFO:tasks.rados.rados.0.smithi033.stdout:3943: done (3 left) 2024-09-21T10:03:51.588 INFO:tasks.rados.rados.0.smithi033.stdout:3944: done (2 left) 2024-09-21T10:03:51.588 INFO:tasks.rados.rados.0.smithi033.stdout:3946: done (1 left) 2024-09-21T10:03:51.588 INFO:tasks.rados.rados.0.smithi033.stdout:3947: rmattr oid 8 current snap is 354 2024-09-21T10:03:51.588 INFO:tasks.rados.rados.0.smithi033.stdout:3947: done (1 left) 2024-09-21T10:03:51.588 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 36 current snap is 354 2024-09-21T10:03:51.588 INFO:tasks.rados.rados.0.smithi033.stdout:3948: seq_num 1246 ranges {0=475136,475136=786432,1261568=417792} 2024-09-21T10:03:51.591 INFO:tasks.rados.rados.0.smithi033.stdout:3948: writing smithi03385152-36 from 0 to 475136 tid 1 2024-09-21T10:03:51.596 INFO:tasks.rados.rados.0.smithi033.stdout:3948: writing smithi03385152-36 from 475136 to 1261568 tid 2 2024-09-21T10:03:51.598 INFO:tasks.rados.rados.0.smithi033.stdout:3948: writing smithi03385152-36 from 1261568 to 1679360 tid 3 2024-09-21T10:03:51.598 INFO:tasks.rados.rados.0.smithi033.stdout:3949: rollback oid 3 current snap is 354 2024-09-21T10:03:51.598 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 3 to 348 2024-09-21T10:03:51.599 INFO:tasks.rados.rados.0.smithi033.stdout:3950: rmattr oid 2 current snap is 354 2024-09-21T10:03:51.599 INFO:tasks.rados.rados.0.smithi033.stdout:3951: read oid 40 snap -1 2024-09-21T10:03:51.599 INFO:tasks.rados.rados.0.smithi033.stdout:3951: expect (ObjNum 1216 snap 351 seq_num 1216) 2024-09-21T10:03:51.599 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 21 current snap is 354 2024-09-21T10:03:51.599 INFO:tasks.rados.rados.0.smithi033.stdout:3952: seq_num 1247 ranges {4210688=491520,4702208=712704,5414912=442368,5857280=49152} 2024-09-21T10:03:51.624 INFO:tasks.rados.rados.0.smithi033.stdout:3952: writing smithi03385152-21 from 4210688 to 4702208 tid 1 2024-09-21T10:03:51.629 INFO:tasks.rados.rados.0.smithi033.stdout:3952: writing smithi03385152-21 from 4702208 to 5414912 tid 2 2024-09-21T10:03:51.631 INFO:tasks.rados.rados.0.smithi033.stdout:3952: writing smithi03385152-21 from 5414912 to 5857280 tid 3 2024-09-21T10:03:51.632 INFO:tasks.rados.rados.0.smithi033.stdout:3952: writing smithi03385152-21 from 5857280 to 5906432 tid 4 2024-09-21T10:03:51.632 INFO:tasks.rados.rados.0.smithi033.stdout:3953: snap_create 2024-09-21T10:03:51.632 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 2 v 2420 (ObjNum 129 snap 0 seq_num 91168720) dirty exists 2024-09-21T10:03:51.632 INFO:tasks.rados.rados.0.smithi033.stdout:3951: expect (ObjNum 1216 snap 351 seq_num 1216) 2024-09-21T10:03:51.783 INFO:tasks.rados.rados.0.smithi033.stdout:3948: finishing write tid 1 to smithi03385152-36 2024-09-21T10:03:51.784 INFO:tasks.rados.rados.0.smithi033.stdout:3945: expect (ObjNum 1191 snap 344 seq_num 1191) 2024-09-21T10:03:51.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:03:51] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.51.0" 2024-09-21T10:03:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[98546]: pgmap v821: 121 pgs: 121 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-09-21T10:03:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[98546]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T10:03:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:51.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[96568]: pgmap v821: 121 pgs: 121 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-09-21T10:03:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[96568]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T10:03:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:51 smithi105 ceph-mon[86649]: pgmap v821: 121 pgs: 121 active+clean; 207 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.0 MiB/s wr, 12 op/s 2024-09-21T10:03:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:51 smithi105 ceph-mon[86649]: osdmap e791: 8 total, 8 up, 8 in 2024-09-21T10:03:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:03:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:52.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:52.181 INFO:tasks.rados.rados.0.smithi033.stdout:3948: finishing write tid 2 to smithi03385152-36 2024-09-21T10:03:52.181 INFO:tasks.rados.rados.0.smithi033.stdout:3948: finishing write tid 3 to smithi03385152-36 2024-09-21T10:03:52.181 INFO:tasks.rados.rados.0.smithi033.stdout:3948: finishing write tid 4 to smithi03385152-36 2024-09-21T10:03:52.181 INFO:tasks.rados.rados.0.smithi033.stdout:3949: finishing rollback tid 0 to smithi03385152-3 2024-09-21T10:03:52.181 INFO:tasks.rados.rados.0.smithi033.stdout:3949: finishing rollback tid 1 to smithi03385152-3 2024-09-21T10:03:52.181 INFO:tasks.rados.rados.0.smithi033.stdout:3949: finishing rollback tid 2 to smithi03385152-3 2024-09-21T10:03:52.181 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 3 v 2591 (ObjNum 1196 snap 345 seq_num 1196) dirty exists 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3948: finishing write tid 5 to smithi03385152-36 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 36 v 2826 (ObjNum 1246 snap 354 seq_num 1246) dirty exists 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3948: left oid 36 (ObjNum 1246 snap 354 seq_num 1246) 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3952: finishing write tid 1 to smithi03385152-21 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3952: finishing write tid 2 to smithi03385152-21 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3952: finishing write tid 3 to smithi03385152-21 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3952: finishing write tid 4 to smithi03385152-21 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3952: finishing write tid 5 to smithi03385152-21 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3952: finishing write tid 6 to smithi03385152-21 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 21 v 2478 (ObjNum 1247 snap 354 seq_num 1247) dirty exists 2024-09-21T10:03:52.182 INFO:tasks.rados.rados.0.smithi033.stdout:3952: left oid 21 (ObjNum 1247 snap 354 seq_num 1247) 2024-09-21T10:03:52.638 INFO:tasks.rados.rados.0.smithi033.stdout:3945: done (6 left) 2024-09-21T10:03:52.638 INFO:tasks.rados.rados.0.smithi033.stdout:3948: done (5 left) 2024-09-21T10:03:52.638 INFO:tasks.rados.rados.0.smithi033.stdout:3949: done (4 left) 2024-09-21T10:03:52.639 INFO:tasks.rados.rados.0.smithi033.stdout:3950: done (3 left) 2024-09-21T10:03:52.639 INFO:tasks.rados.rados.0.smithi033.stdout:3951: done (2 left) 2024-09-21T10:03:52.639 INFO:tasks.rados.rados.0.smithi033.stdout:3952: done (1 left) 2024-09-21T10:03:52.639 INFO:tasks.rados.rados.0.smithi033.stdout:3953: done (0 left) 2024-09-21T10:03:52.639 INFO:tasks.rados.rados.0.smithi033.stdout:3954: snap_create 2024-09-21T10:03:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:52.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:52.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:52.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:52.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:03:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:03:53.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:03:53.586 INFO:tasks.rados.rados.0.smithi033.stdout:3954: done (0 left) 2024-09-21T10:03:53.586 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 32 current snap is 356 2024-09-21T10:03:53.587 INFO:tasks.rados.rados.0.smithi033.stdout:3955: seq_num 1248 ranges {606208=507904,1114112=598016,1712128=606208} 2024-09-21T10:03:53.592 INFO:tasks.rados.rados.0.smithi033.stdout:3955: writing smithi03385152-32 from 606208 to 1114112 tid 1 2024-09-21T10:03:53.596 INFO:tasks.rados.rados.0.smithi033.stdout:3955: writing smithi03385152-32 from 1114112 to 1712128 tid 2 2024-09-21T10:03:53.601 INFO:tasks.rados.rados.0.smithi033.stdout:3955: writing smithi03385152-32 from 1712128 to 2318336 tid 3 2024-09-21T10:03:53.601 INFO:tasks.rados.rados.0.smithi033.stdout:3956: read oid 50 snap 355 2024-09-21T10:03:53.601 INFO:tasks.rados.rados.0.smithi033.stdout:3956: expect deleted 2024-09-21T10:03:53.601 INFO:tasks.rados.rados.0.smithi033.stdout:3957: copy_from oid 45 from oid 16 current snap is 356 2024-09-21T10:03:53.601 INFO:tasks.rados.rados.0.smithi033.stdout:3958: snap_create 2024-09-21T10:03:53.601 INFO:tasks.rados.rados.0.smithi033.stdout:3955: finishing write tid 1 to smithi03385152-32 2024-09-21T10:03:53.601 INFO:tasks.rados.rados.0.smithi033.stdout:3955: finishing write tid 2 to smithi03385152-32 2024-09-21T10:03:53.604 INFO:tasks.rados.rados.0.smithi033.stdout:3955: finishing write tid 3 to smithi03385152-32 2024-09-21T10:03:53.604 INFO:tasks.rados.rados.0.smithi033.stdout:3955: finishing write tid 4 to smithi03385152-32 2024-09-21T10:03:53.605 INFO:tasks.rados.rados.0.smithi033.stdout:3955: finishing write tid 5 to smithi03385152-32 2024-09-21T10:03:53.605 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 32 v 1608 (ObjNum 1248 snap 356 seq_num 1248) dirty exists 2024-09-21T10:03:53.605 INFO:tasks.rados.rados.0.smithi033.stdout:3955: left oid 32 (ObjNum 1248 snap 356 seq_num 1248) 2024-09-21T10:03:53.613 INFO:tasks.rados.rados.0.smithi033.stdout:3957: finishing copy_from racing read to smithi03385152-45 2024-09-21T10:03:53.616 INFO:tasks.rados.rados.0.smithi033.stdout:3957: finishing copy_from to smithi03385152-45 2024-09-21T10:03:53.616 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2757 (ObjNum 1219 snap 351 seq_num 1219) dirty exists 2024-09-21T10:03:53.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:53 smithi033 ceph-mon[98546]: pgmap v823: 121 pgs: 121 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 16 op/s 2024-09-21T10:03:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:53 smithi033 ceph-mon[98546]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T10:03:53.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:53 smithi033 ceph-mon[98546]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T10:03:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:53 smithi033 ceph-mon[96568]: pgmap v823: 121 pgs: 121 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 16 op/s 2024-09-21T10:03:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:53 smithi033 ceph-mon[96568]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T10:03:53.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:53 smithi033 ceph-mon[96568]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T10:03:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:53 smithi105 ceph-mon[86649]: pgmap v823: 121 pgs: 121 active+clean; 210 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.1 MiB/s wr, 16 op/s 2024-09-21T10:03:54.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:53 smithi105 ceph-mon[86649]: osdmap e792: 8 total, 8 up, 8 in 2024-09-21T10:03:54.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:53 smithi105 ceph-mon[86649]: osdmap e793: 8 total, 8 up, 8 in 2024-09-21T10:03:54.598 INFO:tasks.rados.rados.0.smithi033.stdout:3955: done (3 left) 2024-09-21T10:03:54.599 INFO:tasks.rados.rados.0.smithi033.stdout:3956: done (2 left) 2024-09-21T10:03:54.599 INFO:tasks.rados.rados.0.smithi033.stdout:3957: done (1 left) 2024-09-21T10:03:54.599 INFO:tasks.rados.rados.0.smithi033.stdout:3958: done (0 left) 2024-09-21T10:03:54.599 INFO:tasks.rados.rados.0.smithi033.stdout:3959: read oid 34 snap -1 2024-09-21T10:03:54.599 INFO:tasks.rados.rados.0.smithi033.stdout:3959: expect (ObjNum 1209 snap 349 seq_num 1209) 2024-09-21T10:03:54.599 INFO:tasks.rados.rados.0.smithi033.stdout:3960: read oid 7 snap -1 2024-09-21T10:03:54.599 INFO:tasks.rados.rados.0.smithi033.stdout:3960: expect deleted 2024-09-21T10:03:54.600 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 15 current snap is 357 2024-09-21T10:03:54.600 INFO:tasks.rados.rados.0.smithi033.stdout:3961: seq_num 1249 ranges {2023424=524288,2547712=483328,3031040=409600,3440640=311296} 2024-09-21T10:03:54.612 INFO:tasks.rados.rados.0.smithi033.stdout:3961: writing smithi03385152-15 from 2023424 to 2547712 tid 1 2024-09-21T10:03:54.615 INFO:tasks.rados.rados.0.smithi033.stdout:3961: writing smithi03385152-15 from 2547712 to 3031040 tid 2 2024-09-21T10:03:54.618 INFO:tasks.rados.rados.0.smithi033.stdout:3961: writing smithi03385152-15 from 3031040 to 3440640 tid 3 2024-09-21T10:03:54.620 INFO:tasks.rados.rados.0.smithi033.stdout:3961: writing smithi03385152-15 from 3440640 to 3751936 tid 4 2024-09-21T10:03:54.620 INFO:tasks.rados.rados.0.smithi033.stdout:3962: snap_create 2024-09-21T10:03:54.620 INFO:tasks.rados.rados.0.smithi033.stdout:3959: expect (ObjNum 1209 snap 349 seq_num 1209) 2024-09-21T10:03:54.683 INFO:tasks.rados.rados.0.smithi033.stdout:3961: finishing write tid 1 to smithi03385152-15 2024-09-21T10:03:54.683 INFO:tasks.rados.rados.0.smithi033.stdout:3961: finishing write tid 2 to smithi03385152-15 2024-09-21T10:03:54.683 INFO:tasks.rados.rados.0.smithi033.stdout:3961: finishing write tid 3 to smithi03385152-15 2024-09-21T10:03:54.683 INFO:tasks.rados.rados.0.smithi033.stdout:3961: finishing write tid 4 to smithi03385152-15 2024-09-21T10:03:54.683 INFO:tasks.rados.rados.0.smithi033.stdout:3961: finishing write tid 5 to smithi03385152-15 2024-09-21T10:03:54.683 INFO:tasks.rados.rados.0.smithi033.stdout:3961: finishing write tid 6 to smithi03385152-15 2024-09-21T10:03:54.684 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 2488 (ObjNum 1249 snap 357 seq_num 1249) dirty exists 2024-09-21T10:03:54.684 INFO:tasks.rados.rados.0.smithi033.stdout:3961: left oid 15 (ObjNum 1249 snap 357 seq_num 1249) 2024-09-21T10:03:55.599 INFO:tasks.rados.rados.0.smithi033.stdout:3959: done (3 left) 2024-09-21T10:03:55.600 INFO:tasks.rados.rados.0.smithi033.stdout:3960: done (2 left) 2024-09-21T10:03:55.600 INFO:tasks.rados.rados.0.smithi033.stdout:3961: done (1 left) 2024-09-21T10:03:55.600 INFO:tasks.rados.rados.0.smithi033.stdout:3962: done (0 left) 2024-09-21T10:03:55.600 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 27 current snap is 358 2024-09-21T10:03:55.600 INFO:tasks.rados.rados.0.smithi033.stdout:3963: seq_num 1250 ranges {2351104=540672,2891776=770048,3661824=434176} 2024-09-21T10:03:55.615 INFO:tasks.rados.rados.0.smithi033.stdout:3963: writing smithi03385152-27 from 2351104 to 2891776 tid 1 2024-09-21T10:03:55.620 INFO:tasks.rados.rados.0.smithi033.stdout:3963: writing smithi03385152-27 from 2891776 to 3661824 tid 2 2024-09-21T10:03:55.622 INFO:tasks.rados.rados.0.smithi033.stdout:3963: writing smithi03385152-27 from 3661824 to 4096000 tid 3 2024-09-21T10:03:55.623 INFO:tasks.rados.rados.0.smithi033.stdout:3964: delete oid 49 current snap is 358 2024-09-21T10:03:55.623 INFO:tasks.rados.rados.0.smithi033.stdout:3963: finishing write tid 1 to smithi03385152-27 2024-09-21T10:03:55.626 INFO:tasks.rados.rados.0.smithi033.stdout:3963: finishing write tid 2 to smithi03385152-27 2024-09-21T10:03:55.626 INFO:tasks.rados.rados.0.smithi033.stdout:3964: done (1 left) 2024-09-21T10:03:55.626 INFO:tasks.rados.rados.0.smithi033.stdout:3965: delete oid 16 current snap is 358 2024-09-21T10:03:55.628 INFO:tasks.rados.rados.0.smithi033.stdout:3963: finishing write tid 3 to smithi03385152-27 2024-09-21T10:03:55.629 INFO:tasks.rados.rados.0.smithi033.stdout:3963: finishing write tid 4 to smithi03385152-27 2024-09-21T10:03:55.629 INFO:tasks.rados.rados.0.smithi033.stdout:3963: finishing write tid 5 to smithi03385152-27 2024-09-21T10:03:55.629 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 27 v 2528 (ObjNum 1250 snap 358 seq_num 1250) dirty exists 2024-09-21T10:03:55.629 INFO:tasks.rados.rados.0.smithi033.stdout:3963: left oid 27 (ObjNum 1250 snap 358 seq_num 1250) 2024-09-21T10:03:55.630 INFO:tasks.rados.rados.0.smithi033.stdout:3963: done (1 left) 2024-09-21T10:03:55.630 INFO:tasks.rados.rados.0.smithi033.stdout:3965: done (0 left) 2024-09-21T10:03:55.630 INFO:tasks.rados.rados.0.smithi033.stdout:3966: setattr oid 19 current snap is 358 2024-09-21T10:03:55.631 INFO:tasks.rados.rados.0.smithi033.stdout:3967: read oid 50 snap -1 2024-09-21T10:03:55.631 INFO:tasks.rados.rados.0.smithi033.stdout:3967: expect deleted 2024-09-21T10:03:55.631 INFO:tasks.rados.rados.0.smithi033.stdout:3968: setattr oid 14 current snap is 358 2024-09-21T10:03:55.632 INFO:tasks.rados.rados.0.smithi033.stdout:3967: done (2 left) 2024-09-21T10:03:55.632 INFO:tasks.rados.rados.0.smithi033.stdout:3969: copy_from oid 15 from oid 3 current snap is 358 2024-09-21T10:03:55.632 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 42 current snap is 358 2024-09-21T10:03:55.632 INFO:tasks.rados.rados.0.smithi033.stdout:3970: seq_num 1253 ranges {11223040=589824,11812864=417792,12230656=786432} 2024-09-21T10:03:55.697 INFO:tasks.rados.rados.0.smithi033.stdout:3970: writing smithi03385152-42 from 11223040 to 11812864 tid 1 2024-09-21T10:03:55.699 INFO:tasks.rados.rados.0.smithi033.stdout:3970: writing smithi03385152-42 from 11812864 to 12230656 tid 2 2024-09-21T10:03:55.704 INFO:tasks.rados.rados.0.smithi033.stdout:3970: writing smithi03385152-42 from 12230656 to 13017088 tid 3 2024-09-21T10:03:55.704 INFO:tasks.rados.rados.0.smithi033.stdout:3971: delete oid 37 current snap is 358 2024-09-21T10:03:55.704 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 19 v 2496 (ObjNum 1127 snap 325 seq_num 1127) dirty exists 2024-09-21T10:03:55.705 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 14 v 2758 (ObjNum 1067 snap 313 seq_num 1067) dirty exists 2024-09-21T10:03:55.705 INFO:tasks.rados.rados.0.smithi033.stdout:3969: finishing copy_from racing read to smithi03385152-15 2024-09-21T10:03:55.705 INFO:tasks.rados.rados.0.smithi033.stdout:3969: finishing copy_from to smithi03385152-15 2024-09-21T10:03:55.705 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 15 v 2591 (ObjNum 1196 snap 345 seq_num 1196) dirty exists 2024-09-21T10:03:55.705 INFO:tasks.rados.rados.0.smithi033.stdout:3970: finishing write tid 1 to smithi03385152-42 2024-09-21T10:03:55.705 INFO:tasks.rados.rados.0.smithi033.stdout:3970: finishing write tid 2 to smithi03385152-42 2024-09-21T10:03:55.707 INFO:tasks.rados.rados.0.smithi033.stdout:3966: done (4 left) 2024-09-21T10:03:55.707 INFO:tasks.rados.rados.0.smithi033.stdout:3968: done (3 left) 2024-09-21T10:03:55.707 INFO:tasks.rados.rados.0.smithi033.stdout:3969: done (2 left) 2024-09-21T10:03:55.707 INFO:tasks.rados.rados.0.smithi033.stdout:3971: done (1 left) 2024-09-21T10:03:55.707 INFO:tasks.rados.rados.0.smithi033.stdout:3972: read oid 44 snap -1 2024-09-21T10:03:55.708 INFO:tasks.rados.rados.0.smithi033.stdout:3972: expect (ObjNum 1236 snap 354 seq_num 1236) 2024-09-21T10:03:55.708 INFO:tasks.rados.rados.0.smithi033.stdout:3973: read oid 10 snap -1 2024-09-21T10:03:55.708 INFO:tasks.rados.rados.0.smithi033.stdout:3973: expect (ObjNum 1214 snap 351 seq_num 1214) 2024-09-21T10:03:55.708 INFO:tasks.rados.rados.0.smithi033.stdout:3974: read oid 49 snap -1 2024-09-21T10:03:55.708 INFO:tasks.rados.rados.0.smithi033.stdout:3974: expect deleted 2024-09-21T10:03:55.708 INFO:tasks.rados.rados.0.smithi033.stdout:3975: copy_from oid 12 from oid 23 current snap is 358 2024-09-21T10:03:55.708 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 22 current snap is 358 2024-09-21T10:03:55.708 INFO:tasks.rados.rados.0.smithi033.stdout:3976: seq_num 1255 ranges {10412032=622592,11034624=638976,11673600=565248} 2024-09-21T10:03:55.768 INFO:tasks.rados.rados.0.smithi033.stdout:3976: writing smithi03385152-22 from 10412032 to 11034624 tid 1 2024-09-21T10:03:55.771 INFO:tasks.rados.rados.0.smithi033.stdout:3976: writing smithi03385152-22 from 11034624 to 11673600 tid 2 2024-09-21T10:03:55.775 INFO:tasks.rados.rados.0.smithi033.stdout:3976: writing smithi03385152-22 from 11673600 to 12238848 tid 3 2024-09-21T10:03:55.775 INFO:tasks.rados.rados.0.smithi033.stdout:3977: snap_create 2024-09-21T10:03:55.775 INFO:tasks.rados.rados.0.smithi033.stdout:3970: finishing write tid 3 to smithi03385152-42 2024-09-21T10:03:55.775 INFO:tasks.rados.rados.0.smithi033.stdout:3970: finishing write tid 4 to smithi03385152-42 2024-09-21T10:03:55.775 INFO:tasks.rados.rados.0.smithi033.stdout:3970: finishing write tid 5 to smithi03385152-42 2024-09-21T10:03:55.775 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 42 v 2510 (ObjNum 1253 snap 358 seq_num 1253) dirty exists 2024-09-21T10:03:55.775 INFO:tasks.rados.rados.0.smithi033.stdout:3970: left oid 42 (ObjNum 1253 snap 358 seq_num 1253) 2024-09-21T10:03:55.775 INFO:tasks.rados.rados.0.smithi033.stdout:3972: expect (ObjNum 1236 snap 354 seq_num 1236) 2024-09-21T10:03:55.898 INFO:tasks.rados.rados.0.smithi033.stdout:3973: expect (ObjNum 1214 snap 351 seq_num 1214) 2024-09-21T10:03:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:55 smithi033 ceph-mon[98546]: pgmap v826: 121 pgs: 121 active+clean; 207 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 17 op/s 2024-09-21T10:03:55.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:55 smithi033 ceph-mon[98546]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T10:03:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:55 smithi033 ceph-mon[96568]: pgmap v826: 121 pgs: 121 active+clean; 207 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 17 op/s 2024-09-21T10:03:55.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:55 smithi033 ceph-mon[96568]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T10:03:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:55 smithi105 ceph-mon[86649]: pgmap v826: 121 pgs: 121 active+clean; 207 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 3.6 MiB/s wr, 17 op/s 2024-09-21T10:03:56.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:55 smithi105 ceph-mon[86649]: osdmap e794: 8 total, 8 up, 8 in 2024-09-21T10:03:56.416 INFO:tasks.rados.rados.0.smithi033.stdout:3975: finishing copy_from racing read to smithi03385152-12 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:3975: finishing copy_from to smithi03385152-12 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 12 v 2466 (ObjNum 1172 snap 336 seq_num 1172) dirty exists 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:3976: finishing write tid 1 to smithi03385152-22 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:3976: finishing write tid 2 to smithi03385152-22 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:3976: finishing write tid 3 to smithi03385152-22 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:3976: finishing write tid 4 to smithi03385152-22 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:3976: finishing write tid 5 to smithi03385152-22 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 22 v 2762 (ObjNum 1255 snap 358 seq_num 1255) dirty exists 2024-09-21T10:03:56.417 INFO:tasks.rados.rados.0.smithi033.stdout:3976: left oid 22 (ObjNum 1255 snap 358 seq_num 1255) 2024-09-21T10:03:56.609 INFO:tasks.rados.rados.0.smithi033.stdout:3970: done (6 left) 2024-09-21T10:03:56.609 INFO:tasks.rados.rados.0.smithi033.stdout:3972: done (5 left) 2024-09-21T10:03:56.609 INFO:tasks.rados.rados.0.smithi033.stdout:3973: done (4 left) 2024-09-21T10:03:56.609 INFO:tasks.rados.rados.0.smithi033.stdout:3974: done (3 left) 2024-09-21T10:03:56.609 INFO:tasks.rados.rados.0.smithi033.stdout:3975: done (2 left) 2024-09-21T10:03:56.610 INFO:tasks.rados.rados.0.smithi033.stdout:3976: done (1 left) 2024-09-21T10:03:56.610 INFO:tasks.rados.rados.0.smithi033.stdout:3977: done (0 left) 2024-09-21T10:03:56.610 INFO:tasks.rados.rados.0.smithi033.stdout:append oid 5 current snap is 359 2024-09-21T10:03:56.610 INFO:tasks.rados.rados.0.smithi033.stdout:3978: seq_num 1256 ranges {5120000=638976,5758976=524288,6283264=614400,6897664=65536} 2024-09-21T10:03:56.640 INFO:tasks.rados.rados.0.smithi033.stdout:3978: writing smithi03385152-5 from 5120000 to 5758976 tid 1 2024-09-21T10:03:56.643 INFO:tasks.rados.rados.0.smithi033.stdout:3978: writing smithi03385152-5 from 5758976 to 6283264 tid 2 2024-09-21T10:03:56.647 INFO:tasks.rados.rados.0.smithi033.stdout:3978: writing smithi03385152-5 from 6283264 to 6897664 tid 3 2024-09-21T10:03:56.647 INFO:tasks.rados.rados.0.smithi033.stdout:3978: writing smithi03385152-5 from 6897664 to 6963200 tid 4 2024-09-21T10:03:56.647 INFO:tasks.rados.rados.0.smithi033.stdout:3979: delete oid 35 current snap is 359 2024-09-21T10:03:56.647 INFO:tasks.rados.rados.0.smithi033.stdout:3978: finishing write tid 1 to smithi03385152-5 2024-09-21T10:03:56.649 INFO:tasks.rados.rados.0.smithi033.stdout:3978: finishing write tid 2 to smithi03385152-5 2024-09-21T10:03:56.650 INFO:tasks.rados.rados.0.smithi033.stdout:3978: finishing write tid 3 to smithi03385152-5 2024-09-21T10:03:56.650 INFO:tasks.rados.rados.0.smithi033.stdout:3978: finishing write tid 4 to smithi03385152-5 2024-09-21T10:03:56.651 INFO:tasks.rados.rados.0.smithi033.stdout:3979: done (1 left) 2024-09-21T10:03:56.651 INFO:tasks.rados.rados.0.smithi033.stdout:3980: rmattr oid 45 current snap is 359 2024-09-21T10:03:56.651 INFO:tasks.rados.rados.0.smithi033.stdout:3981: read oid 20 snap -1 2024-09-21T10:03:56.651 INFO:tasks.rados.rados.0.smithi033.stdout:3981: expect (ObjNum 1215 snap 351 seq_num 1215) 2024-09-21T10:03:56.651 INFO:tasks.rados.rados.0.smithi033.stdout:3982: read oid 35 snap -1 2024-09-21T10:03:56.652 INFO:tasks.rados.rados.0.smithi033.stdout:3982: expect deleted 2024-09-21T10:03:56.652 INFO:tasks.rados.rados.0.smithi033.stdout:3983: copy_from oid 16 from oid 36 current snap is 359 2024-09-21T10:03:56.652 INFO:tasks.rados.rados.0.smithi033.stdout:3984: snap_remove snap 355 2024-09-21T10:03:56.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:56 smithi033 ceph-mon[98546]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T10:03:56.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:56 smithi033 ceph-mon[96568]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T10:03:57.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:56 smithi105 ceph-mon[86649]: osdmap e795: 8 total, 8 up, 8 in 2024-09-21T10:03:57.618 INFO:tasks.rados.rados.0.smithi033.stdout:3984: done (5 left) 2024-09-21T10:03:57.618 INFO:tasks.rados.rados.0.smithi033.stdout:3985: snap_remove snap 354 2024-09-21T10:03:57.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:57 smithi033 ceph-mon[98546]: pgmap v829: 121 pgs: 121 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 26 op/s 2024-09-21T10:03:57.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:57 smithi033 ceph-mon[98546]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T10:03:57.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:57 smithi033 ceph-mon[96568]: pgmap v829: 121 pgs: 121 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 26 op/s 2024-09-21T10:03:57.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:57 smithi033 ceph-mon[96568]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T10:03:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:57 smithi105 ceph-mon[86649]: pgmap v829: 121 pgs: 121 active+clean; 213 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 26 op/s 2024-09-21T10:03:58.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:57 smithi105 ceph-mon[86649]: osdmap e796: 8 total, 8 up, 8 in 2024-09-21T10:03:58.591 INFO:tasks.rados.rados.0.smithi033.stdout:3985: done (5 left) 2024-09-21T10:03:58.591 INFO:tasks.rados.rados.0.smithi033.stdout:3986: read oid 21 snap 357 2024-09-21T10:03:58.592 INFO:tasks.rados.rados.0.smithi033.stdout:3986: expect (ObjNum 1247 snap 354 seq_num 1247) 2024-09-21T10:03:58.592 INFO:tasks.rados.rados.0.smithi033.stdout:3987: copy_from oid 33 from oid 41 current snap is 359 2024-09-21T10:03:58.592 INFO:tasks.rados.rados.0.smithi033.stdout:3978: finishing write tid 5 to smithi03385152-5 2024-09-21T10:03:58.592 INFO:tasks.rados.rados.0.smithi033.stdout:3978: finishing write tid 6 to smithi03385152-5 2024-09-21T10:03:58.592 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 5 v 2436 (ObjNum 1256 snap 359 seq_num 1256) dirty exists 2024-09-21T10:03:58.592 INFO:tasks.rados.rados.0.smithi033.stdout:3978: left oid 5 (ObjNum 1256 snap 359 seq_num 1256) 2024-09-21T10:03:58.592 INFO:tasks.rados.rados.0.smithi033.stdout:3981: expect (ObjNum 1215 snap 351 seq_num 1215) 2024-09-21T10:03:58.647 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 45 v 2763 (ObjNum 1219 snap 351 seq_num 1219) dirty exists 2024-09-21T10:03:58.647 INFO:tasks.rados.rados.0.smithi033.stdout:3983: finishing copy_from racing read to smithi03385152-16 2024-09-21T10:03:58.647 INFO:tasks.rados.rados.0.smithi033.stdout:3983: finishing copy_from to smithi03385152-16 2024-09-21T10:03:58.647 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 16 v 2826 (ObjNum 1246 snap 354 seq_num 1246) dirty exists 2024-09-21T10:03:58.647 INFO:tasks.rados.rados.0.smithi033.stdout:3978: done (6 left) 2024-09-21T10:03:58.647 INFO:tasks.rados.rados.0.smithi033.stdout:3980: done (5 left) 2024-09-21T10:03:58.647 INFO:tasks.rados.rados.0.smithi033.stdout:3981: done (4 left) 2024-09-21T10:03:58.648 INFO:tasks.rados.rados.0.smithi033.stdout:3982: done (3 left) 2024-09-21T10:03:58.648 INFO:tasks.rados.rados.0.smithi033.stdout:3983: done (2 left) 2024-09-21T10:03:58.648 INFO:tasks.rados.rados.0.smithi033.stdout:3988: snap_create 2024-09-21T10:03:58.648 INFO:tasks.rados.rados.0.smithi033.stdout:3987: finishing copy_from to smithi03385152-33 2024-09-21T10:03:58.648 INFO:tasks.rados.rados.0.smithi033.stdout:3987: got expected ENOENT (src dne) 2024-09-21T10:03:58.648 INFO:tasks.rados.rados.0.smithi033.stdout:3987: finishing copy_from racing read to smithi03385152-33 2024-09-21T10:03:58.648 INFO:tasks.rados.rados.0.smithi033.stdout:3986: expect (ObjNum 1247 snap 354 seq_num 1247) 2024-09-21T10:03:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:58 smithi033 ceph-mon[98546]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T10:03:58.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:58 smithi033 ceph-mon[98546]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T10:03:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:58 smithi033 ceph-mon[96568]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T10:03:58.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:58 smithi033 ceph-mon[96568]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T10:03:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:58 smithi105 ceph-mon[86649]: osdmap e797: 8 total, 8 up, 8 in 2024-09-21T10:03:59.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:58 smithi105 ceph-mon[86649]: osdmap e798: 8 total, 8 up, 8 in 2024-09-21T10:03:59.638 INFO:tasks.rados.rados.0.smithi033.stdout:3986: done (2 left) 2024-09-21T10:03:59.638 INFO:tasks.rados.rados.0.smithi033.stdout:3987: done (1 left) 2024-09-21T10:03:59.638 INFO:tasks.rados.rados.0.smithi033.stdout:3988: done (0 left) 2024-09-21T10:03:59.638 INFO:tasks.rados.rados.0.smithi033.stdout:3989: snap_create 2024-09-21T10:03:59.931 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:03:59 smithi105 ceph-mon[86649]: pgmap v832: 121 pgs: 121 active+clean; 215 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-09-21T10:03:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:03:59 smithi033 ceph-mon[98546]: pgmap v832: 121 pgs: 121 active+clean; 215 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-09-21T10:03:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:03:59 smithi033 ceph-mon[96568]: pgmap v832: 121 pgs: 121 active+clean; 215 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.3 MiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-09-21T10:04:00.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:04:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:04:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:00.646 INFO:tasks.rados.rados.0.smithi033.stdout:3989: done (0 left) 2024-09-21T10:04:00.646 INFO:tasks.rados.rados.0.smithi033.stdout:3990: rollback oid 9 current snap is 361 2024-09-21T10:04:00.646 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 9 to 356 2024-09-21T10:04:00.646 INFO:tasks.rados.rados.0.smithi033.stdout:3991: copy_from oid 11 from oid 31 current snap is 361 2024-09-21T10:04:00.646 INFO:tasks.rados.rados.0.smithi033.stdout:3992: read oid 13 snap -1 2024-09-21T10:04:00.646 INFO:tasks.rados.rados.0.smithi033.stdout:3992: expect (ObjNum 1221 snap 351 seq_num 1221) 2024-09-21T10:04:00.646 INFO:tasks.rados.rados.0.smithi033.stdout:3993: snap_create 2024-09-21T10:04:00.650 INFO:tasks.rados.rados.0.smithi033.stdout:3990: finishing rollback tid 0 to smithi03385152-9 2024-09-21T10:04:00.651 INFO:tasks.rados.rados.0.smithi033.stdout:3990: finishing rollback tid 1 to smithi03385152-9 2024-09-21T10:04:00.653 INFO:tasks.rados.rados.0.smithi033.stdout:3990: finishing rollback tid 2 to smithi03385152-9 2024-09-21T10:04:00.653 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 9 v 2513 (ObjNum 1116 snap 324 seq_num 1116) dirty exists 2024-09-21T10:04:00.658 INFO:tasks.rados.rados.0.smithi033.stdout:3992: expect (ObjNum 1221 snap 351 seq_num 1221) 2024-09-21T10:04:00.724 INFO:tasks.rados.rados.0.smithi033.stdout:3991: finishing copy_from racing read to smithi03385152-11 2024-09-21T10:04:00.728 INFO:tasks.rados.rados.0.smithi033.stdout:3991: finishing copy_from to smithi03385152-11 2024-09-21T10:04:00.728 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 11 v 2822 (ObjNum 1244 snap 354 seq_num 1244) dirty exists 2024-09-21T10:04:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:00 smithi033 ceph-mon[98546]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T10:04:00.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:00 smithi033 ceph-mon[96568]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T10:04:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:00 smithi105 ceph-mon[86649]: osdmap e799: 8 total, 8 up, 8 in 2024-09-21T10:04:01.647 INFO:tasks.rados.rados.0.smithi033.stdout:3990: done (3 left) 2024-09-21T10:04:01.647 INFO:tasks.rados.rados.0.smithi033.stdout:3991: done (2 left) 2024-09-21T10:04:01.647 INFO:tasks.rados.rados.0.smithi033.stdout:3992: done (1 left) 2024-09-21T10:04:01.647 INFO:tasks.rados.rados.0.smithi033.stdout:3993: done (0 left) 2024-09-21T10:04:01.647 INFO:tasks.rados.rados.0.smithi033.stdout:3994: rollback oid 20 current snap is 362 2024-09-21T10:04:01.647 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 20 to 358 2024-09-21T10:04:01.648 INFO:tasks.rados.rados.0.smithi033.stdout:3995: rmattr oid 23 current snap is 362 2024-09-21T10:04:01.648 INFO:tasks.rados.rados.0.smithi033.stdout:3996: rollback oid 6 current snap is 362 2024-09-21T10:04:01.648 INFO:tasks.rados.rados.0.smithi033.stdout:rollback oid 6 to 356 2024-09-21T10:04:01.648 INFO:tasks.rados.rados.0.smithi033.stdout:3997: delete oid 36 current snap is 362 2024-09-21T10:04:01.655 INFO:tasks.rados.rados.0.smithi033.stdout:3996: finishing rollback tid 0 to smithi03385152-6 2024-09-21T10:04:01.655 INFO:tasks.rados.rados.0.smithi033.stdout:3996: finishing rollback tid 1 to smithi03385152-6 2024-09-21T10:04:01.655 INFO:tasks.rados.rados.0.smithi033.stdout:3996: finishing rollback tid 2 to smithi03385152-6 2024-09-21T10:04:01.655 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 6 v 2531 (ObjNum 1242 snap 354 seq_num 1242) dirty exists 2024-09-21T10:04:01.656 INFO:tasks.rados.rados.0.smithi033.stdout:3994: finishing rollback tid 0 to smithi03385152-20 2024-09-21T10:04:01.656 INFO:tasks.rados.rados.0.smithi033.stdout:3994: finishing rollback tid 1 to smithi03385152-20 2024-09-21T10:04:01.656 INFO:tasks.rados.rados.0.smithi033.stdout:3994: finishing rollback tid 2 to smithi03385152-20 2024-09-21T10:04:01.656 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 20 v 2481 (ObjNum 1215 snap 351 seq_num 1215) dirty exists 2024-09-21T10:04:01.657 INFO:tasks.rados.rados.0.smithi033.stdout:3994: done (3 left) 2024-09-21T10:04:01.657 INFO:tasks.rados.rados.0.smithi033.stdout:3996: done (2 left) 2024-09-21T10:04:01.657 INFO:tasks.rados.rados.0.smithi033.stdout:3997: done (1 left) 2024-09-21T10:04:01.657 INFO:tasks.rados.rados.0.smithi033.stdout:3998: delete oid 6 current snap is 362 2024-09-21T10:04:01.657 INFO:tasks.rados.rados.0.smithi033.stdout:update_object_version oid 23 v 2482 (ObjNum 1172 snap 336 seq_num 1172) dirty exists 2024-09-21T10:04:01.658 INFO:tasks.rados.rados.0.smithi033.stdout:3995: done (1 left) 2024-09-21T10:04:01.659 INFO:tasks.rados.rados.0.smithi033.stdout:3998: done (0 left) 2024-09-21T10:04:01.659 INFO:tasks.rados.rados.0.smithi033.stdout:3999: snap_remove snap 348 2024-09-21T10:04:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:04:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:04:01] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-09-21T10:04:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:01 smithi033 ceph-mon[98546]: pgmap v835: 121 pgs: 121 active+clean; 230 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 4.4 MiB/s wr, 15 op/s 2024-09-21T10:04:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:01 smithi033 ceph-mon[98546]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T10:04:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:01 smithi033 ceph-mon[96568]: pgmap v835: 121 pgs: 121 active+clean; 230 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 4.4 MiB/s wr, 15 op/s 2024-09-21T10:04:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:01 smithi033 ceph-mon[96568]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T10:04:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:01 smithi105 ceph-mon[86649]: pgmap v835: 121 pgs: 121 active+clean; 230 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.6 MiB/s rd, 4.4 MiB/s wr, 15 op/s 2024-09-21T10:04:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:01 smithi105 ceph-mon[86649]: osdmap e800: 8 total, 8 up, 8 in 2024-09-21T10:04:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:02.655 INFO:tasks.rados.rados.0.smithi033.stdout:3999: done (0 left) 2024-09-21T10:04:02.659 INFO:tasks.rados.rados.0.smithi033.stderr:0 errors. 2024-09-21T10:04:02.659 INFO:tasks.rados.rados.0.smithi033.stderr: 2024-09-21T10:04:02.664 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:04:02.963 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:02 smithi033 ceph-mon[98546]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T10:04:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:02 smithi033 ceph-mon[96568]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T10:04:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:02 smithi105 ceph-mon[86649]: osdmap e801: 8 total, 8 up, 8 in 2024-09-21T10:04:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:03 smithi033 ceph-mon[98546]: pgmap v838: 121 pgs: 121 active+clean; 232 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 19 op/s 2024-09-21T10:04:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:03 smithi033 ceph-mon[98546]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T10:04:03.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:03 smithi033 ceph-mon[96568]: pgmap v838: 121 pgs: 121 active+clean; 232 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 19 op/s 2024-09-21T10:04:03.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:03 smithi033 ceph-mon[96568]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T10:04:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:03 smithi105 ceph-mon[86649]: pgmap v838: 121 pgs: 121 active+clean; 232 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 14 MiB/s rd, 4.6 MiB/s wr, 19 op/s 2024-09-21T10:04:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:03 smithi105 ceph-mon[86649]: osdmap e802: 8 total, 8 up, 8 in 2024-09-21T10:04:04.864 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T10:04:05.054 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":802,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T10:04:02.643243+0000","last_up_change":"2024-09-21T09:59:49.901411+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:47:19.790061+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:47:22.976967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:48:32.209917+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2024-09-21T09:51:23.126670+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":3,"size":3,"min_size":2,"crush_rule":1,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"802","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":447,"snap_epoch":802,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"teuthologyprofile","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":8192,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":312,"up_from":325,"up_thru":560,"down_at":313,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":189402874},{"type":"v1","addr":"172.21.15.33:6803","nonce":189402874}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":189402874},{"type":"v1","addr":"172.21.15.33:6805","nonce":189402874}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":189402874},{"type":"v1","addr":"172.21.15.33:6809","nonce":189402874}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":189402874},{"type":"v1","addr":"172.21.15.33:6807","nonce":189402874}]},"public_addr":"172.21.15.33:6803/189402874","cluster_addr":"172.21.15.33:6805/189402874","heartbeat_back_addr":"172.21.15.33:6809/189402874","heartbeat_front_addr":"172.21.15.33:6807/189402874","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":346,"up_from":359,"up_thru":560,"down_at":347,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2766504800},{"type":"v1","addr":"172.21.15.33:6811","nonce":2766504800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2766504800},{"type":"v1","addr":"172.21.15.33:6813","nonce":2766504800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2766504800},{"type":"v1","addr":"172.21.15.33:6817","nonce":2766504800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2766504800},{"type":"v1","addr":"172.21.15.33:6815","nonce":2766504800}]},"public_addr":"172.21.15.33:6811/2766504800","cluster_addr":"172.21.15.33:6813/2766504800","heartbeat_back_addr":"172.21.15.33:6817/2766504800","heartbeat_front_addr":"172.21.15.33:6815/2766504800","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":380,"up_from":393,"up_thru":560,"down_at":381,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":586780654},{"type":"v1","addr":"172.21.15.33:6819","nonce":586780654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":586780654},{"type":"v1","addr":"172.21.15.33:6821","nonce":586780654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":586780654},{"type":"v1","addr":"172.21.15.33:6825","nonce":586780654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":586780654},{"type":"v1","addr":"172.21.15.33:6823","nonce":586780654}]},"public_addr":"172.21.15.33:6819/586780654","cluster_addr":"172.21.15.33:6821/586780654","heartbeat_back_addr":"172.21.15.33:6825/586780654","heartbeat_front_addr":"172.21.15.33:6823/586780654","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":413,"up_from":426,"up_thru":560,"down_at":414,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":610865334},{"type":"v1","addr":"172.21.15.33:6827","nonce":610865334}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":610865334},{"type":"v1","addr":"172.21.15.33:6829","nonce":610865334}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":610865334},{"type":"v1","addr":"172.21.15.33:6833","nonce":610865334}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":610865334},{"type":"v1","addr":"172.21.15.33:6831","nonce":610865334}]},"public_addr":"172.21.15.33:6827/610865334","cluster_addr":"172.21.15.33:6829/610865334","heartbeat_back_addr":"172.21.15.33:6833/610865334","heartbeat_front_addr":"172.21.15.33:6831/610865334","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":446,"up_from":460,"up_thru":560,"down_at":447,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":2250129110},{"type":"v1","addr":"172.21.15.105:6801","nonce":2250129110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":2250129110},{"type":"v1","addr":"172.21.15.105:6803","nonce":2250129110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":2250129110},{"type":"v1","addr":"172.21.15.105:6807","nonce":2250129110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":2250129110},{"type":"v1","addr":"172.21.15.105:6805","nonce":2250129110}]},"public_addr":"172.21.15.105:6801/2250129110","cluster_addr":"172.21.15.105:6803/2250129110","heartbeat_back_addr":"172.21.15.105:6807/2250129110","heartbeat_front_addr":"172.21.15.105:6805/2250129110","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":480,"up_from":494,"up_thru":560,"down_at":481,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1966612698},{"type":"v1","addr":"172.21.15.105:6809","nonce":1966612698}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1966612698},{"type":"v1","addr":"172.21.15.105:6811","nonce":1966612698}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1966612698},{"type":"v1","addr":"172.21.15.105:6815","nonce":1966612698}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1966612698},{"type":"v1","addr":"172.21.15.105:6813","nonce":1966612698}]},"public_addr":"172.21.15.105:6809/1966612698","cluster_addr":"172.21.15.105:6811/1966612698","heartbeat_back_addr":"172.21.15.105:6815/1966612698","heartbeat_front_addr":"172.21.15.105:6813/1966612698","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":512,"up_from":525,"up_thru":560,"down_at":513,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":580557017},{"type":"v1","addr":"172.21.15.105:6817","nonce":580557017}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":580557017},{"type":"v1","addr":"172.21.15.105:6819","nonce":580557017}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":580557017},{"type":"v1","addr":"172.21.15.105:6823","nonce":580557017}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":580557017},{"type":"v1","addr":"172.21.15.105:6821","nonce":580557017}]},"public_addr":"172.21.15.105:6817/580557017","cluster_addr":"172.21.15.105:6819/580557017","heartbeat_back_addr":"172.21.15.105:6823/580557017","heartbeat_front_addr":"172.21.15.105:6821/580557017","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":547,"up_from":560,"up_thru":560,"down_at":548,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":3020670744},{"type":"v1","addr":"172.21.15.105:6825","nonce":3020670744}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":3020670744},{"type":"v1","addr":"172.21.15.105:6827","nonce":3020670744}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":3020670744},{"type":"v1","addr":"172.21.15.105:6831","nonce":3020670744}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":3020670744},{"type":"v1","addr":"172.21.15.105:6829","nonce":3020670744}]},"public_addr":"172.21.15.105:6825/3020670744","cluster_addr":"172.21.15.105:6827/3020670744","heartbeat_back_addr":"172.21.15.105:6831/3020670744","heartbeat_front_addr":"172.21.15.105:6829/3020670744","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":312},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":345},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":380},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":413},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":446},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":480},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":512},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":547}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1a","mappings":[{"from":7,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.2f","mappings":[{"from":4,"to":3}]},{"pgid":"3.34","mappings":[{"from":4,"to":2}]},{"pgid":"3.39","mappings":[{"from":0,"to":1}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:6833/2691139879":"2024-09-22T10:00:09.714128+0000","172.21.15.33:0/2645856759":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3947635653":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/150268354":"2024-09-22T09:48:29.268285+0000","172.21.15.105:6835/3560340417":"2024-09-22T09:53:50.171580+0000","172.21.15.105:6834/3560340417":"2024-09-22T09:53:50.171580+0000","172.21.15.33:0/482369733":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/131382457":"2024-09-22T09:52:46.436950+0000","172.21.15.105:0/1108084292":"2024-09-22T09:53:50.171580+0000","172.21.15.33:0/4220506658":"2024-09-22T09:54:30.283006+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6801/244283938":"2024-09-22T09:48:29.268285+0000","172.21.15.33:6800/244283938":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/1301914674":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/2041164136":"2024-09-22T09:52:46.436950+0000","172.21.15.105:0/1100101125":"2024-09-22T09:53:50.171580+0000","172.21.15.33:0/3579553953":"2024-09-22T09:52:46.436950+0000","172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/700678975":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/2740931131":"2024-09-22T09:54:30.283006+0000","172.21.15.33:6801/2967063367":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/2390585401":"2024-09-22T09:52:46.436950+0000","172.21.15.33:6800/2967063367":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/395059673":"2024-09-22T09:52:46.436950+0000","172.21.15.33:6800/289485480":"2024-09-22T09:52:46.436950+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.105:0/2004410743":"2024-09-22T09:53:50.171580+0000","172.21.15.33:6801/289485480":"2024-09-22T09:52:46.436950+0000","172.21.15.105:0/3260429203":"2024-09-22T09:53:50.171580+0000","172.21.15.105:0/253870186":"2024-09-22T09:53:50.171580+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/1606444157":"2024-09-22T09:54:30.283006+0000","172.21.15.105:6832/2691139879":"2024-09-22T10:00:09.714128+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/1156071037":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/3347076479":"2024-09-22T09:48:29.268285+0000","172.21.15.105:0/768914160":"2024-09-22T09:53:50.171580+0000","172.21.15.105:0/4091807878":"2024-09-22T09:53:50.171580+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":434,"length":1},{"begin":440,"length":2}]}],"new_removed_snaps":[{"pool":5,"snaps":[{"begin":434,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:04:05.631 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph pg dump --format=json 2024-09-21T10:04:05.898 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:05 smithi033 ceph-mon[98546]: pgmap v840: 121 pgs: 121 active+clean; 209 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 710 KiB/s wr, 6 op/s 2024-09-21T10:04:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:05 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/938077697' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:05 smithi033 ceph-mon[96568]: pgmap v840: 121 pgs: 121 active+clean; 209 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 710 KiB/s wr, 6 op/s 2024-09-21T10:04:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:05 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/938077697' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:05 smithi105 ceph-mon[86649]: pgmap v840: 121 pgs: 121 active+clean; 209 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 710 KiB/s wr, 6 op/s 2024-09-21T10:04:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:05 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/938077697' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:04:07.966 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T10:04:07.966 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:07 smithi033 ceph-mon[98546]: pgmap v841: 121 pgs: 121 active+clean; 209 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.2 MiB/s wr, 8 op/s 2024-09-21T10:04:07.967 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:07 smithi033 ceph-mon[96568]: pgmap v841: 121 pgs: 121 active+clean; 209 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.2 MiB/s wr, 8 op/s 2024-09-21T10:04:07.970 INFO:teuthology.orchestra.run.smithi033.stderr:dumped all 2024-09-21T10:04:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:07 smithi105 ceph-mon[86649]: pgmap v841: 121 pgs: 121 active+clean; 209 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.2 MiB/s wr, 8 op/s 2024-09-21T10:04:08.524 INFO:teuthology.orchestra.run.smithi033.stdout:{"pg_ready":true,"pg_map":{"version":841,"stamp":"2024-09-21T10:04:06.453738+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":218864637,"num_objects":89,"num_object_clones":22,"num_object_copies":267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":89,"num_whiteouts":6,"num_read":4724,"num_read_kb":5658159,"num_write":9113,"num_write_kb":1896633,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":120,"num_bytes_recovered":368894496,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17273,"ondisk_log_size":17273,"up":363,"acting":363,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1977600,"kb_used_data":417672,"kb_used_omap":255,"kb_used_meta":1559552,"kb_avail":747983616,"statfs":{"total":767960285184,"available":765935222784,"internally_reserved":0,"allocated":427696128,"data_stored":494474887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":261902,"internal_metadata":1596981490},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-22765568,"num_objects":-24,"num_object_clones":-20,"num_object_copies":-72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-24,"num_whiteouts":-2,"num_read":18,"num_read_kb":25372,"num_write":33,"num_write_kb":19744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001585"},"pg_stats":[{"pgid":"3.3f","version":"0'0","reported_seq":998,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.662674+0000","last_change":"2024-09-21T09:59:50.918957+0000","last_active":"2024-09-21T10:04:02.662674+0000","last_peered":"2024-09-21T10:04:02.662674+0000","last_clean":"2024-09-21T10:04:02.662674+0000","last_became_active":"2024-09-21T09:59:50.918577+0000","last_became_peered":"2024-09-21T09:59:50.918577+0000","last_unstale":"2024-09-21T10:04:02.662674+0000","last_undegraded":"2024-09-21T10:04:02.662674+0000","last_fullsized":"2024-09-21T10:04:02.662674+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:35:34.724100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3e","version":"0'0","reported_seq":1204,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.656565+0000","last_change":"2024-09-21T09:59:15.565981+0000","last_active":"2024-09-21T10:04:02.656565+0000","last_peered":"2024-09-21T10:04:02.656565+0000","last_clean":"2024-09-21T10:04:02.656565+0000","last_became_active":"2024-09-21T09:59:15.565204+0000","last_became_peered":"2024-09-21T09:59:15.565204+0000","last_unstale":"2024-09-21T10:04:02.656565+0000","last_undegraded":"2024-09-21T10:04:02.656565+0000","last_fullsized":"2024-09-21T10:04:02.656565+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:08:13.113209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3d","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651526+0000","last_change":"2024-09-21T09:57:00.542339+0000","last_active":"2024-09-21T10:04:02.651526+0000","last_peered":"2024-09-21T10:04:02.651526+0000","last_clean":"2024-09-21T10:04:02.651526+0000","last_became_active":"2024-09-21T09:57:00.541993+0000","last_became_peered":"2024-09-21T09:57:00.541993+0000","last_unstale":"2024-09-21T10:04:02.651526+0000","last_undegraded":"2024-09-21T10:04:02.651526+0000","last_fullsized":"2024-09-21T10:04:02.651526+0000","mapping_epoch":393,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:25:28.776079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.3c","version":"0'0","reported_seq":1071,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651608+0000","last_change":"2024-09-21T09:59:50.931330+0000","last_active":"2024-09-21T10:04:02.651608+0000","last_peered":"2024-09-21T10:04:02.651608+0000","last_clean":"2024-09-21T10:04:02.651608+0000","last_became_active":"2024-09-21T09:59:50.930893+0000","last_became_peered":"2024-09-21T09:59:50.930893+0000","last_unstale":"2024-09-21T10:04:02.651608+0000","last_undegraded":"2024-09-21T10:04:02.651608+0000","last_fullsized":"2024-09-21T10:04:02.651608+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:05:54.056895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3b","version":"0'0","reported_seq":1015,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.661483+0000","last_change":"2024-09-21T09:59:50.925900+0000","last_active":"2024-09-21T10:04:02.661483+0000","last_peered":"2024-09-21T10:04:02.661483+0000","last_clean":"2024-09-21T10:04:02.661483+0000","last_became_active":"2024-09-21T09:59:50.925588+0000","last_became_peered":"2024-09-21T09:59:50.925588+0000","last_unstale":"2024-09-21T10:04:02.661483+0000","last_undegraded":"2024-09-21T10:04:02.661483+0000","last_fullsized":"2024-09-21T10:04:02.661483+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:52:57.095760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.3a","version":"0'0","reported_seq":1051,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651497+0000","last_change":"2024-09-21T09:58:42.603739+0000","last_active":"2024-09-21T10:04:02.651497+0000","last_peered":"2024-09-21T10:04:02.651497+0000","last_clean":"2024-09-21T10:04:02.651497+0000","last_became_active":"2024-09-21T09:58:42.601455+0000","last_became_peered":"2024-09-21T09:58:42.601455+0000","last_unstale":"2024-09-21T10:04:02.651497+0000","last_undegraded":"2024-09-21T10:04:02.651497+0000","last_fullsized":"2024-09-21T10:04:02.651497+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:11:27.439420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.39","version":"0'0","reported_seq":1025,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.661662+0000","last_change":"2024-09-21T09:59:50.930653+0000","last_active":"2024-09-21T10:04:02.661662+0000","last_peered":"2024-09-21T10:04:02.661662+0000","last_clean":"2024-09-21T10:04:02.661662+0000","last_became_active":"2024-09-21T09:59:50.930296+0000","last_became_peered":"2024-09-21T09:59:50.930296+0000","last_unstale":"2024-09-21T10:04:02.661662+0000","last_undegraded":"2024-09-21T10:04:02.661662+0000","last_fullsized":"2024-09-21T10:04:02.661662+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:15.587326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.38","version":"0'0","reported_seq":1114,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882375+0000","last_change":"2024-09-21T09:59:15.577129+0000","last_active":"2024-09-21T10:04:01.882375+0000","last_peered":"2024-09-21T10:04:01.882375+0000","last_clean":"2024-09-21T10:04:01.882375+0000","last_became_active":"2024-09-21T09:59:15.576819+0000","last_became_peered":"2024-09-21T09:59:15.576819+0000","last_unstale":"2024-09-21T10:04:01.882375+0000","last_undegraded":"2024-09-21T10:04:01.882375+0000","last_fullsized":"2024-09-21T10:04:01.882375+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:07:14.692534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.37","version":"0'0","reported_seq":1008,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.662582+0000","last_change":"2024-09-21T09:59:50.925407+0000","last_active":"2024-09-21T10:04:02.662582+0000","last_peered":"2024-09-21T10:04:02.662582+0000","last_clean":"2024-09-21T10:04:02.662582+0000","last_became_active":"2024-09-21T09:59:50.924930+0000","last_became_peered":"2024-09-21T09:59:50.924930+0000","last_unstale":"2024-09-21T10:04:02.662582+0000","last_undegraded":"2024-09-21T10:04:02.662582+0000","last_fullsized":"2024-09-21T10:04:02.662582+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:11:26.520292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.36","version":"0'0","reported_seq":1031,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.656491+0000","last_change":"2024-09-21T09:59:15.577036+0000","last_active":"2024-09-21T10:04:02.656491+0000","last_peered":"2024-09-21T10:04:02.656491+0000","last_clean":"2024-09-21T10:04:02.656491+0000","last_became_active":"2024-09-21T09:59:15.574577+0000","last_became_peered":"2024-09-21T09:59:15.574577+0000","last_unstale":"2024-09-21T10:04:02.656491+0000","last_undegraded":"2024-09-21T10:04:02.656491+0000","last_fullsized":"2024-09-21T10:04:02.656491+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:30.683093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.35","version":"0'0","reported_seq":1031,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.656121+0000","last_change":"2024-09-21T09:59:15.574153+0000","last_active":"2024-09-21T10:04:02.656121+0000","last_peered":"2024-09-21T10:04:02.656121+0000","last_clean":"2024-09-21T10:04:02.656121+0000","last_became_active":"2024-09-21T09:59:15.573845+0000","last_became_peered":"2024-09-21T09:59:15.573845+0000","last_unstale":"2024-09-21T10:04:02.656121+0000","last_undegraded":"2024-09-21T10:04:02.656121+0000","last_fullsized":"2024-09-21T10:04:02.656121+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:00:26.957737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.34","version":"0'0","reported_seq":1171,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.653280+0000","last_change":"2024-09-21T09:59:15.566363+0000","last_active":"2024-09-21T10:04:02.653280+0000","last_peered":"2024-09-21T10:04:02.653280+0000","last_clean":"2024-09-21T10:04:02.653280+0000","last_became_active":"2024-09-21T09:59:15.566037+0000","last_became_peered":"2024-09-21T09:59:15.566037+0000","last_unstale":"2024-09-21T10:04:02.653280+0000","last_undegraded":"2024-09-21T10:04:02.653280+0000","last_fullsized":"2024-09-21T10:04:02.653280+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:33:29.070026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.33","version":"0'0","reported_seq":1136,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882857+0000","last_change":"2024-09-21T09:59:15.578166+0000","last_active":"2024-09-21T10:04:01.882857+0000","last_peered":"2024-09-21T10:04:01.882857+0000","last_clean":"2024-09-21T10:04:01.882857+0000","last_became_active":"2024-09-21T09:59:15.577839+0000","last_became_peered":"2024-09-21T09:59:15.577839+0000","last_unstale":"2024-09-21T10:04:01.882857+0000","last_undegraded":"2024-09-21T10:04:01.882857+0000","last_fullsized":"2024-09-21T10:04:01.882857+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:01:29.338250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.32","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651155+0000","last_change":"2024-09-21T09:58:42.602638+0000","last_active":"2024-09-21T10:04:02.651155+0000","last_peered":"2024-09-21T10:04:02.651155+0000","last_clean":"2024-09-21T10:04:02.651155+0000","last_became_active":"2024-09-21T09:58:42.602300+0000","last_became_peered":"2024-09-21T09:58:42.602300+0000","last_unstale":"2024-09-21T10:04:02.651155+0000","last_undegraded":"2024-09-21T10:04:02.651155+0000","last_fullsized":"2024-09-21T10:04:02.651155+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:40:54.709438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.31","version":"0'0","reported_seq":1215,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.654746+0000","last_change":"2024-09-21T09:58:42.600564+0000","last_active":"2024-09-21T10:04:02.654746+0000","last_peered":"2024-09-21T10:04:02.654746+0000","last_clean":"2024-09-21T10:04:02.654746+0000","last_became_active":"2024-09-21T09:58:42.600226+0000","last_became_peered":"2024-09-21T09:58:42.600226+0000","last_unstale":"2024-09-21T10:04:02.654746+0000","last_undegraded":"2024-09-21T10:04:02.654746+0000","last_fullsized":"2024-09-21T10:04:02.654746+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:39:03.624808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.30","version":"0'0","reported_seq":1052,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651059+0000","last_change":"2024-09-21T09:59:50.930527+0000","last_active":"2024-09-21T10:04:02.651059+0000","last_peered":"2024-09-21T10:04:02.651059+0000","last_clean":"2024-09-21T10:04:02.651059+0000","last_became_active":"2024-09-21T09:59:50.929977+0000","last_became_peered":"2024-09-21T09:59:50.929977+0000","last_unstale":"2024-09-21T10:04:02.651059+0000","last_undegraded":"2024-09-21T10:04:02.651059+0000","last_fullsized":"2024-09-21T10:04:02.651059+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:11:49.529547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2f","version":"0'0","reported_seq":1081,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651962+0000","last_change":"2024-09-21T09:59:50.931849+0000","last_active":"2024-09-21T10:04:02.651962+0000","last_peered":"2024-09-21T10:04:02.651962+0000","last_clean":"2024-09-21T10:04:02.651962+0000","last_became_active":"2024-09-21T09:59:50.931474+0000","last_became_peered":"2024-09-21T09:59:50.931474+0000","last_unstale":"2024-09-21T10:04:02.651962+0000","last_undegraded":"2024-09-21T10:04:02.651962+0000","last_fullsized":"2024-09-21T10:04:02.651962+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:11:00.214844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2e","version":"0'0","reported_seq":1144,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.652271+0000","last_change":"2024-09-21T09:59:50.930260+0000","last_active":"2024-09-21T10:04:02.652271+0000","last_peered":"2024-09-21T10:04:02.652271+0000","last_clean":"2024-09-21T10:04:02.652271+0000","last_became_active":"2024-09-21T09:59:50.929313+0000","last_became_peered":"2024-09-21T09:59:50.929313+0000","last_unstale":"2024-09-21T10:04:02.652271+0000","last_undegraded":"2024-09-21T10:04:02.652271+0000","last_fullsized":"2024-09-21T10:04:02.652271+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:28:29.956539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.2d","version":"0'0","reported_seq":997,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.662168+0000","last_change":"2024-09-21T09:59:50.924619+0000","last_active":"2024-09-21T10:04:02.662168+0000","last_peered":"2024-09-21T10:04:02.662168+0000","last_clean":"2024-09-21T10:04:02.662168+0000","last_became_active":"2024-09-21T09:59:50.924308+0000","last_became_peered":"2024-09-21T09:59:50.924308+0000","last_unstale":"2024-09-21T10:04:02.662168+0000","last_undegraded":"2024-09-21T10:04:02.662168+0000","last_fullsized":"2024-09-21T10:04:02.662168+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:00:22.636878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2c","version":"0'0","reported_seq":1039,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.654859+0000","last_change":"2024-09-21T09:59:15.572035+0000","last_active":"2024-09-21T10:04:02.654859+0000","last_peered":"2024-09-21T10:04:02.654859+0000","last_clean":"2024-09-21T10:04:02.654859+0000","last_became_active":"2024-09-21T09:59:15.571722+0000","last_became_peered":"2024-09-21T09:59:15.571722+0000","last_unstale":"2024-09-21T10:04:02.654859+0000","last_undegraded":"2024-09-21T10:04:02.654859+0000","last_fullsized":"2024-09-21T10:04:02.654859+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:26:31.459575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2b","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650479+0000","last_change":"2024-09-21T09:59:50.918019+0000","last_active":"2024-09-21T10:04:02.650479+0000","last_peered":"2024-09-21T10:04:02.650479+0000","last_clean":"2024-09-21T10:04:02.650479+0000","last_became_active":"2024-09-21T09:59:50.917548+0000","last_became_peered":"2024-09-21T09:59:50.917548+0000","last_unstale":"2024-09-21T10:04:02.650479+0000","last_undegraded":"2024-09-21T10:04:02.650479+0000","last_fullsized":"2024-09-21T10:04:02.650479+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:06:26.144380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2a","version":"0'0","reported_seq":1062,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651216+0000","last_change":"2024-09-21T09:59:50.920561+0000","last_active":"2024-09-21T10:04:02.651216+0000","last_peered":"2024-09-21T10:04:02.651216+0000","last_clean":"2024-09-21T10:04:02.651216+0000","last_became_active":"2024-09-21T09:59:50.920268+0000","last_became_peered":"2024-09-21T09:59:50.920268+0000","last_unstale":"2024-09-21T10:04:02.651216+0000","last_undegraded":"2024-09-21T10:04:02.651216+0000","last_fullsized":"2024-09-21T10:04:02.651216+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:14:19.574641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.29","version":"0'0","reported_seq":1136,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882469+0000","last_change":"2024-09-21T09:58:08.512227+0000","last_active":"2024-09-21T10:04:01.882469+0000","last_peered":"2024-09-21T10:04:01.882469+0000","last_clean":"2024-09-21T10:04:01.882469+0000","last_became_active":"2024-09-21T09:58:08.511915+0000","last_became_peered":"2024-09-21T09:58:08.511915+0000","last_unstale":"2024-09-21T10:04:01.882469+0000","last_undegraded":"2024-09-21T10:04:01.882469+0000","last_fullsized":"2024-09-21T10:04:01.882469+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:33:07.025926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.28","version":"0'0","reported_seq":1077,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651180+0000","last_change":"2024-09-21T09:58:42.601676+0000","last_active":"2024-09-21T10:04:02.651180+0000","last_peered":"2024-09-21T10:04:02.651180+0000","last_clean":"2024-09-21T10:04:02.651180+0000","last_became_active":"2024-09-21T09:58:42.601325+0000","last_became_peered":"2024-09-21T09:58:42.601325+0000","last_unstale":"2024-09-21T10:04:02.651180+0000","last_undegraded":"2024-09-21T10:04:02.651180+0000","last_fullsized":"2024-09-21T10:04:02.651180+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:10:37.869833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.27","version":"0'0","reported_seq":1129,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.883245+0000","last_change":"2024-09-21T09:58:08.509966+0000","last_active":"2024-09-21T10:04:01.883245+0000","last_peered":"2024-09-21T10:04:01.883245+0000","last_clean":"2024-09-21T10:04:01.883245+0000","last_became_active":"2024-09-21T09:58:08.509656+0000","last_became_peered":"2024-09-21T09:58:08.509656+0000","last_unstale":"2024-09-21T10:04:01.883245+0000","last_undegraded":"2024-09-21T10:04:01.883245+0000","last_fullsized":"2024-09-21T10:04:01.883245+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:54:40.592818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.26","version":"0'0","reported_seq":1033,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.655398+0000","last_change":"2024-09-21T09:59:50.915995+0000","last_active":"2024-09-21T10:04:02.655398+0000","last_peered":"2024-09-21T10:04:02.655398+0000","last_clean":"2024-09-21T10:04:02.655398+0000","last_became_active":"2024-09-21T09:59:50.915649+0000","last_became_peered":"2024-09-21T09:59:50.915649+0000","last_unstale":"2024-09-21T10:04:02.655398+0000","last_undegraded":"2024-09-21T10:04:02.655398+0000","last_fullsized":"2024-09-21T10:04:02.655398+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:12:15.083685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.25","version":"0'0","reported_seq":1215,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.659170+0000","last_change":"2024-09-21T09:57:34.564118+0000","last_active":"2024-09-21T10:04:02.659170+0000","last_peered":"2024-09-21T10:04:02.659170+0000","last_clean":"2024-09-21T10:04:02.659170+0000","last_became_active":"2024-09-21T09:57:34.563802+0000","last_became_peered":"2024-09-21T09:57:34.563802+0000","last_unstale":"2024-09-21T10:04:02.659170+0000","last_undegraded":"2024-09-21T10:04:02.659170+0000","last_fullsized":"2024-09-21T10:04:02.659170+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:31:16.584655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.24","version":"0'0","reported_seq":1144,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882930+0000","last_change":"2024-09-21T09:58:08.508740+0000","last_active":"2024-09-21T10:04:01.882930+0000","last_peered":"2024-09-21T10:04:01.882930+0000","last_clean":"2024-09-21T10:04:01.882930+0000","last_became_active":"2024-09-21T09:58:08.508393+0000","last_became_peered":"2024-09-21T09:58:08.508393+0000","last_unstale":"2024-09-21T10:04:01.882930+0000","last_undegraded":"2024-09-21T10:04:01.882930+0000","last_fullsized":"2024-09-21T10:04:01.882930+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:49:07.647653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.23","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651104+0000","last_change":"2024-09-21T09:58:08.515479+0000","last_active":"2024-09-21T10:04:02.651104+0000","last_peered":"2024-09-21T10:04:02.651104+0000","last_clean":"2024-09-21T10:04:02.651104+0000","last_became_active":"2024-09-21T09:58:08.514958+0000","last_became_peered":"2024-09-21T09:58:08.514958+0000","last_unstale":"2024-09-21T10:04:02.651104+0000","last_undegraded":"2024-09-21T10:04:02.651104+0000","last_fullsized":"2024-09-21T10:04:02.651104+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:15:41.655211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.22","version":"0'0","reported_seq":1077,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651304+0000","last_change":"2024-09-21T09:58:42.603826+0000","last_active":"2024-09-21T10:04:02.651304+0000","last_peered":"2024-09-21T10:04:02.651304+0000","last_clean":"2024-09-21T10:04:02.651304+0000","last_became_active":"2024-09-21T09:58:42.600695+0000","last_became_peered":"2024-09-21T09:58:42.600695+0000","last_unstale":"2024-09-21T10:04:02.651304+0000","last_undegraded":"2024-09-21T10:04:02.651304+0000","last_fullsized":"2024-09-21T10:04:02.651304+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:57:46.323812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.21","version":"0'0","reported_seq":1144,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650954+0000","last_change":"2024-09-21T09:59:15.563814+0000","last_active":"2024-09-21T10:04:02.650954+0000","last_peered":"2024-09-21T10:04:02.650954+0000","last_clean":"2024-09-21T10:04:02.650954+0000","last_became_active":"2024-09-21T09:59:15.563414+0000","last_became_peered":"2024-09-21T09:59:15.563414+0000","last_unstale":"2024-09-21T10:04:02.650954+0000","last_undegraded":"2024-09-21T10:04:02.650954+0000","last_fullsized":"2024-09-21T10:04:02.650954+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:53:33.176445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.20","version":"0'0","reported_seq":1031,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.655674+0000","last_change":"2024-09-21T09:59:15.570280+0000","last_active":"2024-09-21T10:04:02.655674+0000","last_peered":"2024-09-21T10:04:02.655674+0000","last_clean":"2024-09-21T10:04:02.655674+0000","last_became_active":"2024-09-21T09:59:15.569298+0000","last_became_peered":"2024-09-21T09:59:15.569298+0000","last_unstale":"2024-09-21T10:04:02.655674+0000","last_undegraded":"2024-09-21T10:04:02.655674+0000","last_fullsized":"2024-09-21T10:04:02.655674+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:29:13.137620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1a","version":"0'0","reported_seq":1113,"reported_epoch":800,"state":"active+clean","last_fresh":"2024-09-21T10:04:00.636859+0000","last_change":"2024-09-21T09:58:08.507980+0000","last_active":"2024-09-21T10:04:00.636859+0000","last_peered":"2024-09-21T10:04:00.636859+0000","last_clean":"2024-09-21T10:04:00.636859+0000","last_became_active":"2024-09-21T09:58:08.506499+0000","last_became_peered":"2024-09-21T09:58:08.506499+0000","last_unstale":"2024-09-21T10:04:00.636859+0000","last_undegraded":"2024-09-21T10:04:00.636859+0000","last_fullsized":"2024-09-21T10:04:00.636859+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:15:58.160120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1b","version":"581'2","reported_seq":1122,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882990+0000","last_change":"2024-09-21T09:59:50.923817+0000","last_active":"2024-09-21T10:04:01.882990+0000","last_peered":"2024-09-21T10:04:01.882990+0000","last_clean":"2024-09-21T10:04:01.882990+0000","last_became_active":"2024-09-21T09:59:50.923402+0000","last_became_peered":"2024-09-21T09:59:50.923402+0000","last_unstale":"2024-09-21T10:04:01.882990+0000","last_undegraded":"2024-09-21T10:04:01.882990+0000","last_fullsized":"2024-09-21T10:04:01.882990+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:28:58.484874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650657+0000","last_change":"2024-09-21T09:59:50.920332+0000","last_active":"2024-09-21T10:04:02.650657+0000","last_peered":"2024-09-21T10:04:02.650657+0000","last_clean":"2024-09-21T10:04:02.650657+0000","last_became_active":"2024-09-21T09:59:50.919432+0000","last_became_peered":"2024-09-21T09:59:50.919432+0000","last_unstale":"2024-09-21T10:04:02.650657+0000","last_undegraded":"2024-09-21T10:04:02.650657+0000","last_fullsized":"2024-09-21T10:04:02.650657+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:31.764888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1a","version":"581'2","reported_seq":1053,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.655157+0000","last_change":"2024-09-21T09:59:15.563957+0000","last_active":"2024-09-21T10:04:02.655157+0000","last_peered":"2024-09-21T10:04:02.655157+0000","last_clean":"2024-09-21T10:04:02.655157+0000","last_became_active":"2024-09-21T09:59:15.563253+0000","last_became_peered":"2024-09-21T09:59:15.563253+0000","last_unstale":"2024-09-21T10:04:02.655157+0000","last_undegraded":"2024-09-21T10:04:02.655157+0000","last_fullsized":"2024-09-21T10:04:02.655157+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:08:30.140636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.19","version":"0'0","reported_seq":1146,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882272+0000","last_change":"2024-09-21T09:59:15.573713+0000","last_active":"2024-09-21T10:04:01.882272+0000","last_peered":"2024-09-21T10:04:01.882272+0000","last_clean":"2024-09-21T10:04:01.882272+0000","last_became_active":"2024-09-21T09:59:15.573210+0000","last_became_peered":"2024-09-21T09:59:15.573210+0000","last_unstale":"2024-09-21T10:04:01.882272+0000","last_undegraded":"2024-09-21T10:04:01.882272+0000","last_fullsized":"2024-09-21T10:04:01.882272+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:49:26.504595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,0],"acting":[3,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1142,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882378+0000","last_change":"2024-09-21T09:57:34.568391+0000","last_active":"2024-09-21T10:04:01.882378+0000","last_peered":"2024-09-21T10:04:01.882378+0000","last_clean":"2024-09-21T10:04:01.882378+0000","last_became_active":"2024-09-21T09:57:34.568022+0000","last_became_peered":"2024-09-21T09:57:34.568022+0000","last_unstale":"2024-09-21T10:04:01.882378+0000","last_undegraded":"2024-09-21T10:04:01.882378+0000","last_fullsized":"2024-09-21T10:04:01.882378+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:41:24.697593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.18","version":"0'0","reported_seq":1066,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651060+0000","last_change":"2024-09-21T09:59:50.930261+0000","last_active":"2024-09-21T10:04:02.651060+0000","last_peered":"2024-09-21T10:04:02.651060+0000","last_clean":"2024-09-21T10:04:02.651060+0000","last_became_active":"2024-09-21T09:59:50.929601+0000","last_became_peered":"2024-09-21T09:59:50.929601+0000","last_unstale":"2024-09-21T10:04:02.651060+0000","last_undegraded":"2024-09-21T10:04:02.651060+0000","last_fullsized":"2024-09-21T10:04:02.651060+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:55:54.029785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1215,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.654645+0000","last_change":"2024-09-21T09:58:08.506107+0000","last_active":"2024-09-21T10:04:02.654645+0000","last_peered":"2024-09-21T10:04:02.654645+0000","last_clean":"2024-09-21T10:04:02.654645+0000","last_became_active":"2024-09-21T09:58:08.505737+0000","last_became_peered":"2024-09-21T09:58:08.505737+0000","last_unstale":"2024-09-21T10:04:02.654645+0000","last_undegraded":"2024-09-21T10:04:02.654645+0000","last_fullsized":"2024-09-21T10:04:02.654645+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:00:54.193971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.17","version":"0'0","reported_seq":1034,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.654884+0000","last_change":"2024-09-21T09:59:15.576202+0000","last_active":"2024-09-21T10:04:02.654884+0000","last_peered":"2024-09-21T10:04:02.654884+0000","last_clean":"2024-09-21T10:04:02.654884+0000","last_became_active":"2024-09-21T09:59:15.575908+0000","last_became_peered":"2024-09-21T09:59:15.575908+0000","last_unstale":"2024-09-21T10:04:02.654884+0000","last_undegraded":"2024-09-21T10:04:02.654884+0000","last_fullsized":"2024-09-21T10:04:02.654884+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:34:52.562445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1045,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651944+0000","last_change":"2024-09-21T09:59:50.921232+0000","last_active":"2024-09-21T10:04:02.651944+0000","last_peered":"2024-09-21T10:04:02.651944+0000","last_clean":"2024-09-21T10:04:02.651944+0000","last_became_active":"2024-09-21T09:59:50.920943+0000","last_became_peered":"2024-09-21T09:59:50.920943+0000","last_unstale":"2024-09-21T10:04:02.651944+0000","last_undegraded":"2024-09-21T10:04:02.651944+0000","last_fullsized":"2024-09-21T10:04:02.651944+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:19:32.565991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.16","version":"0'0","reported_seq":1082,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.652016+0000","last_change":"2024-09-21T09:59:15.570674+0000","last_active":"2024-09-21T10:04:02.652016+0000","last_peered":"2024-09-21T10:04:02.652016+0000","last_clean":"2024-09-21T10:04:02.652016+0000","last_became_active":"2024-09-21T09:59:15.570392+0000","last_became_peered":"2024-09-21T09:59:15.570392+0000","last_unstale":"2024-09-21T10:04:02.652016+0000","last_undegraded":"2024-09-21T10:04:02.652016+0000","last_fullsized":"2024-09-21T10:04:02.652016+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:15:02.705517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651646+0000","last_change":"2024-09-21T09:58:42.603128+0000","last_active":"2024-09-21T10:04:02.651646+0000","last_peered":"2024-09-21T10:04:02.651646+0000","last_clean":"2024-09-21T10:04:02.651646+0000","last_became_active":"2024-09-21T09:58:42.602785+0000","last_became_peered":"2024-09-21T09:58:42.602785+0000","last_unstale":"2024-09-21T10:04:02.651646+0000","last_undegraded":"2024-09-21T10:04:02.651646+0000","last_fullsized":"2024-09-21T10:04:02.651646+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:26:10.368025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.15","version":"57'3","reported_seq":1213,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.659364+0000","last_change":"2024-09-21T09:58:42.602637+0000","last_active":"2024-09-21T10:04:02.659364+0000","last_peered":"2024-09-21T10:04:02.659364+0000","last_clean":"2024-09-21T10:04:02.659364+0000","last_became_active":"2024-09-21T09:58:42.602253+0000","last_became_peered":"2024-09-21T09:58:42.602253+0000","last_unstale":"2024-09-21T10:04:02.659364+0000","last_undegraded":"2024-09-21T10:04:02.659364+0000","last_fullsized":"2024-09-21T10:04:02.659364+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:37:07.276732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"0'0","reported_seq":1090,"reported_epoch":800,"state":"active+clean","last_fresh":"2024-09-21T10:04:00.637523+0000","last_change":"2024-09-21T09:59:50.921808+0000","last_active":"2024-09-21T10:04:00.637523+0000","last_peered":"2024-09-21T10:04:00.637523+0000","last_clean":"2024-09-21T10:04:00.637523+0000","last_became_active":"2024-09-21T09:59:50.921519+0000","last_became_peered":"2024-09-21T09:59:50.921519+0000","last_unstale":"2024-09-21T10:04:00.637523+0000","last_undegraded":"2024-09-21T10:04:00.637523+0000","last_fullsized":"2024-09-21T10:04:00.637523+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:55:39.389854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.14","version":"581'29","reported_seq":1069,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.655045+0000","last_change":"2024-09-21T09:59:15.575404+0000","last_active":"2024-09-21T10:04:02.655045+0000","last_peered":"2024-09-21T10:04:02.655045+0000","last_clean":"2024-09-21T10:04:02.655045+0000","last_became_active":"2024-09-21T09:59:15.575092+0000","last_became_peered":"2024-09-21T09:59:15.575092+0000","last_unstale":"2024-09-21T10:04:02.655045+0000","last_undegraded":"2024-09-21T10:04:02.655045+0000","last_fullsized":"2024-09-21T10:04:02.655045+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":29,"log_dups_size":0,"ondisk_log_size":29,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:18.630079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5816,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":14,"num_write":28,"num_write_kb":25,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"0'0","reported_seq":1015,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.661944+0000","last_change":"2024-09-21T09:59:50.927733+0000","last_active":"2024-09-21T10:04:02.661944+0000","last_peered":"2024-09-21T10:04:02.661944+0000","last_clean":"2024-09-21T10:04:02.661944+0000","last_became_active":"2024-09-21T09:59:50.927392+0000","last_became_peered":"2024-09-21T09:59:50.927392+0000","last_unstale":"2024-09-21T10:04:02.661944+0000","last_undegraded":"2024-09-21T10:04:02.661944+0000","last_fullsized":"2024-09-21T10:04:02.661944+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:15:09.778283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.13","version":"57'2","reported_seq":1248,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650756+0000","last_change":"2024-09-21T09:58:08.514426+0000","last_active":"2024-09-21T10:04:02.650756+0000","last_peered":"2024-09-21T10:04:02.650756+0000","last_clean":"2024-09-21T10:04:02.650756+0000","last_became_active":"2024-09-21T09:58:08.514036+0000","last_became_peered":"2024-09-21T09:58:08.514036+0000","last_unstale":"2024-09-21T10:04:02.650756+0000","last_undegraded":"2024-09-21T10:04:02.650756+0000","last_fullsized":"2024-09-21T10:04:02.650756+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:52:56.969913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650955+0000","last_change":"2024-09-21T09:59:50.916072+0000","last_active":"2024-09-21T10:04:02.650955+0000","last_peered":"2024-09-21T10:04:02.650955+0000","last_clean":"2024-09-21T10:04:02.650955+0000","last_became_active":"2024-09-21T09:59:50.915752+0000","last_became_peered":"2024-09-21T09:59:50.915752+0000","last_unstale":"2024-09-21T10:04:02.650955+0000","last_undegraded":"2024-09-21T10:04:02.650955+0000","last_fullsized":"2024-09-21T10:04:02.650955+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:21.223397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.12","version":"0'0","reported_seq":1074,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651598+0000","last_change":"2024-09-21T09:59:50.919883+0000","last_active":"2024-09-21T10:04:02.651598+0000","last_peered":"2024-09-21T10:04:02.651598+0000","last_clean":"2024-09-21T10:04:02.651598+0000","last_became_active":"2024-09-21T09:59:50.919581+0000","last_became_peered":"2024-09-21T09:59:50.919581+0000","last_unstale":"2024-09-21T10:04:02.651598+0000","last_undegraded":"2024-09-21T10:04:02.651598+0000","last_fullsized":"2024-09-21T10:04:02.651598+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:30:06.221693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"0'0","reported_seq":1015,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.661172+0000","last_change":"2024-09-21T09:59:50.926859+0000","last_active":"2024-09-21T10:04:02.661172+0000","last_peered":"2024-09-21T10:04:02.661172+0000","last_clean":"2024-09-21T10:04:02.661172+0000","last_became_active":"2024-09-21T09:59:50.926259+0000","last_became_peered":"2024-09-21T09:59:50.926259+0000","last_unstale":"2024-09-21T10:04:02.661172+0000","last_undegraded":"2024-09-21T10:04:02.661172+0000","last_fullsized":"2024-09-21T10:04:02.661172+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:03:04.204186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.11","version":"0'0","reported_seq":1042,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.655571+0000","last_change":"2024-09-21T09:59:15.570009+0000","last_active":"2024-09-21T10:04:02.655571+0000","last_peered":"2024-09-21T10:04:02.655571+0000","last_clean":"2024-09-21T10:04:02.655571+0000","last_became_active":"2024-09-21T09:59:15.568778+0000","last_became_peered":"2024-09-21T09:59:15.568778+0000","last_unstale":"2024-09-21T10:04:02.655571+0000","last_undegraded":"2024-09-21T10:04:02.655571+0000","last_fullsized":"2024-09-21T10:04:02.655571+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:23:35.106859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"0'0","reported_seq":1031,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.655778+0000","last_change":"2024-09-21T09:59:15.569766+0000","last_active":"2024-09-21T10:04:02.655778+0000","last_peered":"2024-09-21T10:04:02.655778+0000","last_clean":"2024-09-21T10:04:02.655778+0000","last_became_active":"2024-09-21T09:59:15.566166+0000","last_became_peered":"2024-09-21T09:59:15.566166+0000","last_unstale":"2024-09-21T10:04:02.655778+0000","last_undegraded":"2024-09-21T10:04:02.655778+0000","last_fullsized":"2024-09-21T10:04:02.655778+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:00:22.110927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.10","version":"57'8","reported_seq":1192,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.652196+0000","last_change":"2024-09-21T09:57:00.543681+0000","last_active":"2024-09-21T10:04:02.652196+0000","last_peered":"2024-09-21T10:04:02.652196+0000","last_clean":"2024-09-21T10:04:02.652196+0000","last_became_active":"2024-09-21T09:57:00.542492+0000","last_became_peered":"2024-09-21T09:57:00.542492+0000","last_unstale":"2024-09-21T10:04:02.652196+0000","last_undegraded":"2024-09-21T10:04:02.652196+0000","last_fullsized":"2024-09-21T10:04:02.652196+0000","mapping_epoch":393,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:05:37.518544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":997,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.661632+0000","last_change":"2024-09-21T09:59:50.926565+0000","last_active":"2024-09-21T10:04:02.661632+0000","last_peered":"2024-09-21T10:04:02.661632+0000","last_clean":"2024-09-21T10:04:02.661632+0000","last_became_active":"2024-09-21T09:59:50.926303+0000","last_became_peered":"2024-09-21T09:59:50.926303+0000","last_unstale":"2024-09-21T10:04:02.661632+0000","last_undegraded":"2024-09-21T10:04:02.661632+0000","last_fullsized":"2024-09-21T10:04:02.661632+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:45:52.956677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"802'1700","reported_seq":4737,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.679462+0000","last_change":"2024-09-21T10:04:02.680761+0000","last_active":"2024-09-21T10:04:02.679462+0000","last_peered":"2024-09-21T10:04:02.679462+0000","last_clean":"2024-09-21T10:04:02.679462+0000","last_became_active":"2024-09-21T09:57:00.539600+0000","last_became_peered":"2024-09-21T09:57:00.539600+0000","last_unstale":"2024-09-21T10:04:02.679462+0000","last_undegraded":"2024-09-21T10:04:02.679462+0000","last_fullsized":"2024-09-21T10:04:02.679462+0000","mapping_epoch":393,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":1700,"log_dups_size":0,"ondisk_log_size":1700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:51:53.458148+0000","scrub_duration":0,"objects_trimmed":7,"snaptrim_duration":0.028281661,"stat_sum":{"num_bytes":20652032,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":358,"num_read_kb":407255,"num_write":928,"num_write_kb":196760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":32497664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"2.f","version":"57'1","reported_seq":1085,"reported_epoch":800,"state":"active+clean","last_fresh":"2024-09-21T10:04:00.637517+0000","last_change":"2024-09-21T09:59:50.929595+0000","last_active":"2024-09-21T10:04:00.637517+0000","last_peered":"2024-09-21T10:04:00.637517+0000","last_clean":"2024-09-21T10:04:00.637517+0000","last_became_active":"2024-09-21T09:59:50.929060+0000","last_became_peered":"2024-09-21T09:59:50.929060+0000","last_unstale":"2024-09-21T10:04:00.637517+0000","last_undegraded":"2024-09-21T10:04:00.637517+0000","last_fullsized":"2024-09-21T10:04:00.637517+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:30:18.465476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"0'0","reported_seq":1015,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.662227+0000","last_change":"2024-09-21T09:59:50.928543+0000","last_active":"2024-09-21T10:04:02.662227+0000","last_peered":"2024-09-21T10:04:02.662227+0000","last_clean":"2024-09-21T10:04:02.662227+0000","last_became_active":"2024-09-21T09:59:50.928266+0000","last_became_peered":"2024-09-21T09:59:50.928266+0000","last_unstale":"2024-09-21T10:04:02.662227+0000","last_undegraded":"2024-09-21T10:04:02.662227+0000","last_fullsized":"2024-09-21T10:04:02.662227+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:18:41.055394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"802'1410","reported_seq":4183,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.674294+0000","last_change":"2024-09-21T10:04:02.675264+0000","last_active":"2024-09-21T10:04:02.674294+0000","last_peered":"2024-09-21T10:04:02.674294+0000","last_clean":"2024-09-21T10:04:02.674294+0000","last_became_active":"2024-09-21T09:59:50.929862+0000","last_became_peered":"2024-09-21T09:59:50.929862+0000","last_unstale":"2024-09-21T10:04:02.674294+0000","last_undegraded":"2024-09-21T10:04:02.674294+0000","last_fullsized":"2024-09-21T10:04:02.674294+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":1410,"log_dups_size":0,"ondisk_log_size":1410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:53:54.322628+0000","scrub_duration":0,"objects_trimmed":2,"snaptrim_duration":0.013889914999999999,"stat_sum":{"num_bytes":19144704,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":1,"num_read":362,"num_read_kb":696499,"num_write":732,"num_write_kb":157280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":23486464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"2.e","version":"57'2","reported_seq":1153,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.652416+0000","last_change":"2024-09-21T09:59:50.930448+0000","last_active":"2024-09-21T10:04:02.652416+0000","last_peered":"2024-09-21T10:04:02.652416+0000","last_clean":"2024-09-21T10:04:02.652416+0000","last_became_active":"2024-09-21T09:59:50.929787+0000","last_became_peered":"2024-09-21T09:59:50.929787+0000","last_unstale":"2024-09-21T10:04:02.652416+0000","last_undegraded":"2024-09-21T10:04:02.652416+0000","last_fullsized":"2024-09-21T10:04:02.652416+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:55:27.856268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,7],"acting":[2,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"0'0","reported_seq":1015,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.662840+0000","last_change":"2024-09-21T09:59:50.926551+0000","last_active":"2024-09-21T10:04:02.662840+0000","last_peered":"2024-09-21T10:04:02.662840+0000","last_clean":"2024-09-21T10:04:02.662840+0000","last_became_active":"2024-09-21T09:59:50.926236+0000","last_became_peered":"2024-09-21T09:59:50.926236+0000","last_unstale":"2024-09-21T10:04:02.662840+0000","last_undegraded":"2024-09-21T10:04:02.662840+0000","last_fullsized":"2024-09-21T10:04:02.662840+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:24:19.365452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.a","version":"802'829","reported_seq":3914,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.678394+0000","last_change":"2024-09-21T10:04:02.680191+0000","last_active":"2024-09-21T10:04:02.678394+0000","last_peered":"2024-09-21T10:04:02.678394+0000","last_clean":"2024-09-21T10:04:02.678394+0000","last_became_active":"2024-09-21T09:58:08.511567+0000","last_became_peered":"2024-09-21T09:58:08.511567+0000","last_unstale":"2024-09-21T10:04:02.678394+0000","last_undegraded":"2024-09-21T10:04:02.678394+0000","last_fullsized":"2024-09-21T10:04:02.678394+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":829,"log_dups_size":0,"ondisk_log_size":829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:32.458985+0000","scrub_duration":0,"objects_trimmed":2,"snaptrim_duration":0.026901120000000001,"stat_sum":{"num_bytes":13598720,"num_objects":4,"num_object_clones":1,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":201,"num_read_kb":258867,"num_write":435,"num_write_kb":107296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":9101312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"2.d","version":"57'1","reported_seq":1222,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.659482+0000","last_change":"2024-09-21T09:58:08.503888+0000","last_active":"2024-09-21T10:04:02.659482+0000","last_peered":"2024-09-21T10:04:02.659482+0000","last_clean":"2024-09-21T10:04:02.659482+0000","last_became_active":"2024-09-21T09:58:08.503545+0000","last_became_peered":"2024-09-21T09:58:08.503545+0000","last_unstale":"2024-09-21T10:04:02.659482+0000","last_undegraded":"2024-09-21T10:04:02.659482+0000","last_fullsized":"2024-09-21T10:04:02.659482+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:44:46.372936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1071,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651737+0000","last_change":"2024-09-21T09:59:15.568164+0000","last_active":"2024-09-21T10:04:02.651737+0000","last_peered":"2024-09-21T10:04:02.651737+0000","last_clean":"2024-09-21T10:04:02.651737+0000","last_became_active":"2024-09-21T09:59:15.567888+0000","last_became_peered":"2024-09-21T09:59:15.567888+0000","last_unstale":"2024-09-21T10:04:02.651737+0000","last_undegraded":"2024-09-21T10:04:02.651737+0000","last_fullsized":"2024-09-21T10:04:02.651737+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:47:01.422124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.b","version":"791'292","reported_seq":3365,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.652160+0000","last_change":"2024-09-21T10:04:02.652787+0000","last_active":"2024-09-21T10:04:02.652160+0000","last_peered":"2024-09-21T10:04:02.652160+0000","last_clean":"2024-09-21T10:04:02.652160+0000","last_became_active":"2024-09-21T09:58:42.607816+0000","last_became_peered":"2024-09-21T09:58:42.607816+0000","last_unstale":"2024-09-21T10:04:02.652160+0000","last_undegraded":"2024-09-21T10:04:02.652160+0000","last_fullsized":"2024-09-21T10:04:02.652160+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":292,"log_dups_size":0,"ondisk_log_size":292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T09:51:34.731031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085470800000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":79,"num_read_kb":162386,"num_write":136,"num_write_kb":46416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":10256384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"2.c","version":"0'0","reported_seq":1178,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.653540+0000","last_change":"2024-09-21T09:58:42.609125+0000","last_active":"2024-09-21T10:04:02.653540+0000","last_peered":"2024-09-21T10:04:02.653540+0000","last_clean":"2024-09-21T10:04:02.653540+0000","last_became_active":"2024-09-21T09:58:42.608810+0000","last_became_peered":"2024-09-21T09:58:42.608810+0000","last_unstale":"2024-09-21T10:04:02.653540+0000","last_undegraded":"2024-09-21T10:04:02.653540+0000","last_fullsized":"2024-09-21T10:04:02.653540+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:45.764107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"0'0","reported_seq":997,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.662450+0000","last_change":"2024-09-21T09:59:50.926038+0000","last_active":"2024-09-21T10:04:02.662450+0000","last_peered":"2024-09-21T10:04:02.662450+0000","last_clean":"2024-09-21T10:04:02.662450+0000","last_became_active":"2024-09-21T09:59:50.925727+0000","last_became_peered":"2024-09-21T09:59:50.925727+0000","last_unstale":"2024-09-21T10:04:02.662450+0000","last_undegraded":"2024-09-21T10:04:02.662450+0000","last_fullsized":"2024-09-21T10:04:02.662450+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:39:19.328637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.4","version":"802'1192","reported_seq":3917,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.672218+0000","last_change":"2024-09-21T10:04:02.673393+0000","last_active":"2024-09-21T10:04:02.672218+0000","last_peered":"2024-09-21T10:04:02.672218+0000","last_clean":"2024-09-21T10:04:02.672218+0000","last_became_active":"2024-09-21T09:59:50.925659+0000","last_became_peered":"2024-09-21T09:59:50.925659+0000","last_unstale":"2024-09-21T10:04:02.672218+0000","last_undegraded":"2024-09-21T10:04:02.672218+0000","last_fullsized":"2024-09-21T10:04:02.672218+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":1192,"log_dups_size":0,"ondisk_log_size":1192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:25:52.125785+0000","scrub_duration":0,"objects_trimmed":2,"snaptrim_duration":0.01134571,"stat_sum":{"num_bytes":11083776,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":1,"num_read":239,"num_read_kb":284164,"num_write":714,"num_write_kb":119192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":27082752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"4.5","version":"0'0","reported_seq":1031,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.656367+0000","last_change":"2024-09-21T09:59:50.921340+0000","last_active":"2024-09-21T10:04:02.656367+0000","last_peered":"2024-09-21T10:04:02.656367+0000","last_clean":"2024-09-21T10:04:02.656367+0000","last_became_active":"2024-09-21T09:59:50.920946+0000","last_became_peered":"2024-09-21T09:59:50.920946+0000","last_unstale":"2024-09-21T10:04:02.656367+0000","last_undegraded":"2024-09-21T10:04:02.656367+0000","last_fullsized":"2024-09-21T10:04:02.656367+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:36:00.296586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046213800000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.0","version":"263'160","reported_seq":1328,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.660660+0000","last_change":"2024-09-21T09:59:50.922970+0000","last_active":"2024-09-21T10:04:02.660660+0000","last_peered":"2024-09-21T10:04:02.660660+0000","last_clean":"2024-09-21T10:04:02.660660+0000","last_became_active":"2024-09-21T09:59:50.922553+0000","last_became_peered":"2024-09-21T09:59:50.922553+0000","last_unstale":"2024-09-21T10:04:02.660660+0000","last_undegraded":"2024-09-21T10:04:02.660660+0000","last_fullsized":"2024-09-21T10:04:02.660660+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:44:57.819716+0000","last_clean_scrub_stamp":"2024-09-21T09:44:57.819716+0000","objects_scrubbed":0,"log_size":160,"log_dups_size":0,"ondisk_log_size":160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:27:11.383190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":840,"num_read_kb":1709,"num_write":269,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1082,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651344+0000","last_change":"2024-09-21T09:59:50.932204+0000","last_active":"2024-09-21T10:04:02.651344+0000","last_peered":"2024-09-21T10:04:02.651344+0000","last_clean":"2024-09-21T10:04:02.651344+0000","last_became_active":"2024-09-21T09:59:50.931711+0000","last_became_peered":"2024-09-21T09:59:50.931711+0000","last_unstale":"2024-09-21T10:04:02.651344+0000","last_undegraded":"2024-09-21T10:04:02.651344+0000","last_fullsized":"2024-09-21T10:04:02.651344+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:56:35.274231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1115,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882614+0000","last_change":"2024-09-21T09:59:15.581072+0000","last_active":"2024-09-21T10:04:01.882614+0000","last_peered":"2024-09-21T10:04:01.882614+0000","last_clean":"2024-09-21T10:04:01.882614+0000","last_became_active":"2024-09-21T09:59:15.580806+0000","last_became_peered":"2024-09-21T09:59:15.580806+0000","last_unstale":"2024-09-21T10:04:01.882614+0000","last_undegraded":"2024-09-21T10:04:01.882614+0000","last_fullsized":"2024-09-21T10:04:01.882614+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:19:59.985708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.7","version":"802'2851","reported_seq":5663,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.681695+0000","last_change":"2024-09-21T10:04:02.682513+0000","last_active":"2024-09-21T10:04:02.681695+0000","last_peered":"2024-09-21T10:04:02.681695+0000","last_clean":"2024-09-21T10:04:02.681695+0000","last_became_active":"2024-09-21T09:58:42.610007+0000","last_became_peered":"2024-09-21T09:58:42.610007+0000","last_unstale":"2024-09-21T10:04:02.681695+0000","last_undegraded":"2024-09-21T10:04:02.681695+0000","last_fullsized":"2024-09-21T10:04:02.681695+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":2851,"log_dups_size":0,"ondisk_log_size":2851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:10:51.970148+0000","scrub_duration":0,"objects_trimmed":5,"snaptrim_duration":0.030168039000000001,"stat_sum":{"num_bytes":36913152,"num_objects":10,"num_object_clones":3,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":2,"num_read":602,"num_read_kb":1110895,"num_write":1598,"num_write_kb":362296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":102318080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"4.6","version":"0'0","reported_seq":1208,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650753+0000","last_change":"2024-09-21T09:57:00.550552+0000","last_active":"2024-09-21T10:04:02.650753+0000","last_peered":"2024-09-21T10:04:02.650753+0000","last_clean":"2024-09-21T10:04:02.650753+0000","last_became_active":"2024-09-21T09:57:00.550169+0000","last_became_peered":"2024-09-21T09:57:00.550169+0000","last_unstale":"2024-09-21T10:04:02.650753+0000","last_undegraded":"2024-09-21T10:04:02.650753+0000","last_fullsized":"2024-09-21T10:04:02.650753+0000","mapping_epoch":393,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:49:24.985917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1000,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.660507+0000","last_change":"2024-09-21T09:59:50.928997+0000","last_active":"2024-09-21T10:04:02.660507+0000","last_peered":"2024-09-21T10:04:02.660507+0000","last_clean":"2024-09-21T10:04:02.660507+0000","last_became_active":"2024-09-21T09:59:50.928593+0000","last_became_peered":"2024-09-21T09:59:50.928593+0000","last_unstale":"2024-09-21T10:04:02.660507+0000","last_undegraded":"2024-09-21T10:04:02.660507+0000","last_fullsized":"2024-09-21T10:04:02.660507+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:16.460082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650409+0000","last_change":"2024-09-21T09:58:08.511502+0000","last_active":"2024-09-21T10:04:02.650409+0000","last_peered":"2024-09-21T10:04:02.650409+0000","last_clean":"2024-09-21T10:04:02.650409+0000","last_became_active":"2024-09-21T09:58:08.511151+0000","last_became_peered":"2024-09-21T09:58:08.511151+0000","last_unstale":"2024-09-21T10:04:02.650409+0000","last_undegraded":"2024-09-21T10:04:02.650409+0000","last_fullsized":"2024-09-21T10:04:02.650409+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:55:00.864213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"802'410","reported_seq":3489,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.664204+0000","last_change":"2024-09-21T10:04:02.665301+0000","last_active":"2024-09-21T10:04:02.664204+0000","last_peered":"2024-09-21T10:04:02.664204+0000","last_clean":"2024-09-21T10:04:02.664204+0000","last_became_active":"2024-09-21T09:58:42.609310+0000","last_became_peered":"2024-09-21T09:58:42.609310+0000","last_unstale":"2024-09-21T10:04:02.664204+0000","last_undegraded":"2024-09-21T10:04:02.664204+0000","last_fullsized":"2024-09-21T10:04:02.664204+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":410,"log_dups_size":0,"ondisk_log_size":410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:51:55.753481+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.012551651,"stat_sum":{"num_bytes":2318336,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":91,"num_read_kb":150543,"num_write":237,"num_write_kb":22776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4808704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"4.7","version":"0'0","reported_seq":1202,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.659283+0000","last_change":"2024-09-21T09:58:42.611950+0000","last_active":"2024-09-21T10:04:02.659283+0000","last_peered":"2024-09-21T10:04:02.659283+0000","last_clean":"2024-09-21T10:04:02.659283+0000","last_became_active":"2024-09-21T09:58:42.611416+0000","last_became_peered":"2024-09-21T09:58:42.611416+0000","last_unstale":"2024-09-21T10:04:02.659283+0000","last_undegraded":"2024-09-21T10:04:02.659283+0000","last_fullsized":"2024-09-21T10:04:02.659283+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:38:42.163205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000471278,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1178,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651090+0000","last_change":"2024-09-21T09:57:34.575163+0000","last_active":"2024-09-21T10:04:02.651090+0000","last_peered":"2024-09-21T10:04:02.651090+0000","last_clean":"2024-09-21T10:04:02.651090+0000","last_became_active":"2024-09-21T09:57:34.574859+0000","last_became_peered":"2024-09-21T09:57:34.574859+0000","last_unstale":"2024-09-21T10:04:02.651090+0000","last_undegraded":"2024-09-21T10:04:02.651090+0000","last_fullsized":"2024-09-21T10:04:02.651090+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:06:28.600006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1215,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.655331+0000","last_change":"2024-09-21T09:59:15.572643+0000","last_active":"2024-09-21T10:04:02.655331+0000","last_peered":"2024-09-21T10:04:02.655331+0000","last_clean":"2024-09-21T10:04:02.655331+0000","last_became_active":"2024-09-21T09:59:15.572221+0000","last_became_peered":"2024-09-21T09:59:15.572221+0000","last_unstale":"2024-09-21T10:04:02.655331+0000","last_undegraded":"2024-09-21T10:04:02.655331+0000","last_fullsized":"2024-09-21T10:04:02.655331+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:10:58.067908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.5","version":"802'1082","reported_seq":4234,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.669359+0000","last_change":"2024-09-21T10:04:02.671040+0000","last_active":"2024-09-21T10:04:02.669359+0000","last_peered":"2024-09-21T10:04:02.669359+0000","last_clean":"2024-09-21T10:04:02.669359+0000","last_became_active":"2024-09-21T09:58:08.515095+0000","last_became_peered":"2024-09-21T09:58:08.515095+0000","last_unstale":"2024-09-21T10:04:02.669359+0000","last_undegraded":"2024-09-21T10:04:02.669359+0000","last_fullsized":"2024-09-21T10:04:02.669359+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":1082,"log_dups_size":0,"ondisk_log_size":1082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:32:16.599587+0000","scrub_duration":0,"objects_trimmed":2,"snaptrim_duration":0.018992689,"stat_sum":{"num_bytes":16252928,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":214,"num_read_kb":288240,"num_write":424,"num_write_kb":120328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":26591232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"4.4","version":"0'0","reported_seq":1184,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.657816+0000","last_change":"2024-09-21T09:57:34.561993+0000","last_active":"2024-09-21T10:04:02.657816+0000","last_peered":"2024-09-21T10:04:02.657816+0000","last_clean":"2024-09-21T10:04:02.657816+0000","last_became_active":"2024-09-21T09:57:34.561684+0000","last_became_peered":"2024-09-21T09:57:34.561684+0000","last_unstale":"2024-09-21T10:04:02.657816+0000","last_undegraded":"2024-09-21T10:04:02.657816+0000","last_fullsized":"2024-09-21T10:04:02.657816+0000","mapping_epoch":426,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:13:35.533615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1048,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651428+0000","last_change":"2024-09-21T09:59:15.570091+0000","last_active":"2024-09-21T10:04:02.651428+0000","last_peered":"2024-09-21T10:04:02.651428+0000","last_clean":"2024-09-21T10:04:02.651428+0000","last_became_active":"2024-09-21T09:59:15.569561+0000","last_became_peered":"2024-09-21T09:59:15.569561+0000","last_unstale":"2024-09-21T10:04:02.651428+0000","last_undegraded":"2024-09-21T10:04:02.651428+0000","last_fullsized":"2024-09-21T10:04:02.651428+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:36:06.757979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1080,"reported_epoch":800,"state":"active+clean","last_fresh":"2024-09-21T10:04:00.636995+0000","last_change":"2024-09-21T09:59:15.563738+0000","last_active":"2024-09-21T10:04:00.636995+0000","last_peered":"2024-09-21T10:04:00.636995+0000","last_clean":"2024-09-21T10:04:00.636995+0000","last_became_active":"2024-09-21T09:59:15.563435+0000","last_became_peered":"2024-09-21T09:59:15.563435+0000","last_unstale":"2024-09-21T10:04:00.636995+0000","last_undegraded":"2024-09-21T10:04:00.636995+0000","last_fullsized":"2024-09-21T10:04:00.636995+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T11:49:41.507376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.3","version":"802'855","reported_seq":4055,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.665766+0000","last_change":"2024-09-21T10:04:02.667311+0000","last_active":"2024-09-21T10:04:02.665766+0000","last_peered":"2024-09-21T10:04:02.665766+0000","last_clean":"2024-09-21T10:04:02.665766+0000","last_became_active":"2024-09-21T09:59:15.567813+0000","last_became_peered":"2024-09-21T09:59:15.567813+0000","last_unstale":"2024-09-21T10:04:02.665766+0000","last_undegraded":"2024-09-21T10:04:02.665766+0000","last_fullsized":"2024-09-21T10:04:02.665766+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":855,"log_dups_size":0,"ondisk_log_size":855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:13:37.140437+0000","scrub_duration":0,"objects_trimmed":3,"snaptrim_duration":0.015775128999999999,"stat_sum":{"num_bytes":1327104,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":213,"num_read_kb":309014,"num_write":351,"num_write_kb":75944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":19349504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"4.2","version":"66'2","reported_seq":1186,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.656726+0000","last_change":"2024-09-21T09:58:42.601526+0000","last_active":"2024-09-21T10:04:02.656726+0000","last_peered":"2024-09-21T10:04:02.656726+0000","last_clean":"2024-09-21T10:04:02.656726+0000","last_became_active":"2024-09-21T09:58:42.601161+0000","last_became_peered":"2024-09-21T09:58:42.601161+0000","last_unstale":"2024-09-21T10:04:02.656726+0000","last_undegraded":"2024-09-21T10:04:02.656726+0000","last_fullsized":"2024-09-21T10:04:02.656726+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:51:45.966001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00056908600000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1207,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.657658+0000","last_change":"2024-09-21T09:59:50.919945+0000","last_active":"2024-09-21T10:04:02.657658+0000","last_peered":"2024-09-21T10:04:02.657658+0000","last_clean":"2024-09-21T10:04:02.657658+0000","last_became_active":"2024-09-21T09:59:50.919633+0000","last_became_peered":"2024-09-21T09:59:50.919633+0000","last_unstale":"2024-09-21T10:04:02.657658+0000","last_undegraded":"2024-09-21T10:04:02.657658+0000","last_fullsized":"2024-09-21T10:04:02.657658+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:47:40.991435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1077,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651796+0000","last_change":"2024-09-21T09:58:42.604057+0000","last_active":"2024-09-21T10:04:02.651796+0000","last_peered":"2024-09-21T10:04:02.651796+0000","last_clean":"2024-09-21T10:04:02.651796+0000","last_became_active":"2024-09-21T09:58:42.603520+0000","last_became_peered":"2024-09-21T09:58:42.603520+0000","last_unstale":"2024-09-21T10:04:02.651796+0000","last_undegraded":"2024-09-21T10:04:02.651796+0000","last_fullsized":"2024-09-21T10:04:02.651796+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:51:30.429133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.2","version":"802'621","reported_seq":3457,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.664335+0000","last_change":"2024-09-21T10:04:02.665230+0000","last_active":"2024-09-21T10:04:02.664335+0000","last_peered":"2024-09-21T10:04:02.664335+0000","last_clean":"2024-09-21T10:04:02.664335+0000","last_became_active":"2024-09-21T09:59:15.570750+0000","last_became_peered":"2024-09-21T09:59:15.570750+0000","last_unstale":"2024-09-21T10:04:02.664335+0000","last_undegraded":"2024-09-21T10:04:02.664335+0000","last_fullsized":"2024-09-21T10:04:02.664335+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":621,"log_dups_size":0,"ondisk_log_size":621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:47:34.943746+0000","scrub_duration":0,"objects_trimmed":1,"snaptrim_duration":0.0097559169999999994,"stat_sum":{"num_bytes":3776512,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":153,"num_read_kb":255056,"num_write":285,"num_write_kb":71432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":42573824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"4.3","version":"64'1","reported_seq":1209,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650947+0000","last_change":"2024-09-21T09:59:50.917991+0000","last_active":"2024-09-21T10:04:02.650947+0000","last_peered":"2024-09-21T10:04:02.650947+0000","last_clean":"2024-09-21T10:04:02.650947+0000","last_became_active":"2024-09-21T09:59:50.917556+0000","last_became_peered":"2024-09-21T09:59:50.917556+0000","last_unstale":"2024-09-21T10:04:02.650947+0000","last_undegraded":"2024-09-21T10:04:02.650947+0000","last_fullsized":"2024-09-21T10:04:02.650947+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:22:48.877422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1019,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.662321+0000","last_change":"2024-09-21T09:59:50.926948+0000","last_active":"2024-09-21T10:04:02.662321+0000","last_peered":"2024-09-21T10:04:02.662321+0000","last_clean":"2024-09-21T10:04:02.662321+0000","last_became_active":"2024-09-21T09:59:50.926499+0000","last_became_peered":"2024-09-21T09:59:50.926499+0000","last_unstale":"2024-09-21T10:04:02.662321+0000","last_undegraded":"2024-09-21T10:04:02.662321+0000","last_fullsized":"2024-09-21T10:04:02.662321+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:37:18.990835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1215,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.657233+0000","last_change":"2024-09-21T09:58:42.602918+0000","last_active":"2024-09-21T10:04:02.657233+0000","last_peered":"2024-09-21T10:04:02.657233+0000","last_clean":"2024-09-21T10:04:02.657233+0000","last_became_active":"2024-09-21T09:58:42.602618+0000","last_became_peered":"2024-09-21T09:58:42.602618+0000","last_unstale":"2024-09-21T10:04:02.657233+0000","last_undegraded":"2024-09-21T10:04:02.657233+0000","last_fullsized":"2024-09-21T10:04:02.657233+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:10:14.405339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.1","version":"798'1016","reported_seq":3953,"reported_epoch":800,"state":"active+clean","last_fresh":"2024-09-21T10:04:00.637098+0000","last_change":"2024-09-21T10:03:58.594595+0000","last_active":"2024-09-21T10:04:00.637098+0000","last_peered":"2024-09-21T10:04:00.637098+0000","last_clean":"2024-09-21T10:04:00.637098+0000","last_became_active":"2024-09-21T09:59:50.919754+0000","last_became_peered":"2024-09-21T09:59:50.919754+0000","last_unstale":"2024-09-21T10:04:00.637098+0000","last_undegraded":"2024-09-21T10:04:00.637098+0000","last_fullsized":"2024-09-21T10:04:00.637098+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":1016,"log_dups_size":0,"ondisk_log_size":1016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:50:39.144139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0079854769999999995,"stat_sum":{"num_bytes":10330112,"num_objects":5,"num_object_clones":2,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":227,"num_read_kb":275403,"num_write":452,"num_write_kb":100808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":7938048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1b8","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":1142,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882246+0000","last_change":"2024-09-21T09:59:50.919820+0000","last_active":"2024-09-21T10:04:01.882246+0000","last_peered":"2024-09-21T10:04:01.882246+0000","last_clean":"2024-09-21T10:04:01.882246+0000","last_became_active":"2024-09-21T09:59:50.919476+0000","last_became_peered":"2024-09-21T09:59:50.919476+0000","last_unstale":"2024-09-21T10:04:01.882246+0000","last_undegraded":"2024-09-21T10:04:01.882246+0000","last_fullsized":"2024-09-21T10:04:01.882246+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:13:34.992907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055753599999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"585'2","reported_seq":1223,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.657032+0000","last_change":"2024-09-21T09:59:15.565719+0000","last_active":"2024-09-21T10:04:02.657032+0000","last_peered":"2024-09-21T10:04:02.657032+0000","last_clean":"2024-09-21T10:04:02.657032+0000","last_became_active":"2024-09-21T09:59:15.565303+0000","last_became_peered":"2024-09-21T09:59:15.565303+0000","last_unstale":"2024-09-21T10:04:02.657032+0000","last_undegraded":"2024-09-21T10:04:02.657032+0000","last_fullsized":"2024-09-21T10:04:02.657032+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:39:17.881606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":2,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"0'0","reported_seq":1144,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.883312+0000","last_change":"2024-09-21T09:59:50.923408+0000","last_active":"2024-09-21T10:04:01.883312+0000","last_peered":"2024-09-21T10:04:01.883312+0000","last_clean":"2024-09-21T10:04:01.883312+0000","last_became_active":"2024-09-21T09:59:50.923079+0000","last_became_peered":"2024-09-21T09:59:50.923079+0000","last_unstale":"2024-09-21T10:04:01.883312+0000","last_undegraded":"2024-09-21T10:04:01.883312+0000","last_fullsized":"2024-09-21T10:04:01.883312+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:12:36.086412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.0","version":"785'345","reported_seq":3330,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882715+0000","last_change":"2024-09-21T10:03:58.593611+0000","last_active":"2024-09-21T10:04:01.882715+0000","last_peered":"2024-09-21T10:04:01.882715+0000","last_clean":"2024-09-21T10:04:01.882715+0000","last_became_active":"2024-09-21T09:58:08.510421+0000","last_became_peered":"2024-09-21T09:58:08.510421+0000","last_unstale":"2024-09-21T10:04:01.882715+0000","last_undegraded":"2024-09-21T10:04:01.882715+0000","last_fullsized":"2024-09-21T10:04:01.882715+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":345,"log_dups_size":0,"ondisk_log_size":345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:30:12.067184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00084703700000000003,"stat_sum":{"num_bytes":3260416,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":1,"num_read":67,"num_read_kb":63310,"num_write":168,"num_write_kb":27616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[{"start":"1b8","length":"2"}]},{"pgid":"4.1","version":"0'0","reported_seq":1101,"reported_epoch":800,"state":"active+clean","last_fresh":"2024-09-21T10:04:00.637386+0000","last_change":"2024-09-21T09:59:15.561122+0000","last_active":"2024-09-21T10:04:00.637386+0000","last_peered":"2024-09-21T10:04:00.637386+0000","last_clean":"2024-09-21T10:04:00.637386+0000","last_became_active":"2024-09-21T09:59:15.560825+0000","last_became_peered":"2024-09-21T09:59:15.560825+0000","last_unstale":"2024-09-21T10:04:00.637386+0000","last_undegraded":"2024-09-21T10:04:00.637386+0000","last_fullsized":"2024-09-21T10:04:00.637386+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:48:32.280781+0000","last_clean_scrub_stamp":"2024-09-21T09:48:32.280781+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:21:12.236957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"57'2","reported_seq":1014,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.654706+0000","last_change":"2024-09-21T09:59:50.930181+0000","last_active":"2024-09-21T10:04:02.654706+0000","last_peered":"2024-09-21T10:04:02.654706+0000","last_clean":"2024-09-21T10:04:02.654706+0000","last_became_active":"2024-09-21T09:59:50.929834+0000","last_became_peered":"2024-09-21T09:59:50.929834+0000","last_unstale":"2024-09-21T10:04:02.654706+0000","last_undegraded":"2024-09-21T10:04:02.654706+0000","last_fullsized":"2024-09-21T10:04:02.654706+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:27:42.127347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.650642+0000","last_change":"2024-09-21T09:58:08.511589+0000","last_active":"2024-09-21T10:04:02.650642+0000","last_peered":"2024-09-21T10:04:02.650642+0000","last_clean":"2024-09-21T10:04:02.650642+0000","last_became_active":"2024-09-21T09:58:08.511256+0000","last_became_peered":"2024-09-21T09:58:08.511256+0000","last_unstale":"2024-09-21T10:04:02.650642+0000","last_undegraded":"2024-09-21T10:04:02.650642+0000","last_fullsized":"2024-09-21T10:04:02.650642+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T18:21:51.312952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.f","version":"802'2381","reported_seq":5182,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.675645+0000","last_change":"2024-09-21T10:04:02.677577+0000","last_active":"2024-09-21T10:04:02.675645+0000","last_peered":"2024-09-21T10:04:02.675645+0000","last_clean":"2024-09-21T10:04:02.675645+0000","last_became_active":"2024-09-21T09:59:15.579286+0000","last_became_peered":"2024-09-21T09:59:15.579286+0000","last_unstale":"2024-09-21T10:04:02.675645+0000","last_undegraded":"2024-09-21T10:04:02.675645+0000","last_fullsized":"2024-09-21T10:04:02.675645+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":2381,"log_dups_size":0,"ondisk_log_size":2381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:35:03.671538+0000","scrub_duration":0,"objects_trimmed":4,"snaptrim_duration":0.025364525999999998,"stat_sum":{"num_bytes":34922496,"num_objects":10,"num_object_clones":3,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":492,"num_read_kb":568879,"num_write":1202,"num_write_kb":267528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":25657344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"2.8","version":"0'0","reported_seq":999,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.660676+0000","last_change":"2024-09-21T09:59:50.928661+0000","last_active":"2024-09-21T10:04:02.660676+0000","last_peered":"2024-09-21T10:04:02.660676+0000","last_clean":"2024-09-21T10:04:02.660676+0000","last_became_active":"2024-09-21T09:59:50.928371+0000","last_became_peered":"2024-09-21T09:59:50.928371+0000","last_unstale":"2024-09-21T10:04:02.660676+0000","last_undegraded":"2024-09-21T10:04:02.660676+0000","last_fullsized":"2024-09-21T10:04:02.660676+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:17:44.013268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1107,"reported_epoch":800,"state":"active+clean","last_fresh":"2024-09-21T10:04:00.637613+0000","last_change":"2024-09-21T09:59:50.922020+0000","last_active":"2024-09-21T10:04:00.637613+0000","last_peered":"2024-09-21T10:04:00.637613+0000","last_clean":"2024-09-21T10:04:00.637613+0000","last_became_active":"2024-09-21T09:59:50.921715+0000","last_became_peered":"2024-09-21T09:59:50.921715+0000","last_unstale":"2024-09-21T10:04:00.637613+0000","last_undegraded":"2024-09-21T10:04:00.637613+0000","last_fullsized":"2024-09-21T10:04:00.637613+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:04:32.329946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"791'390","reported_seq":3344,"reported_epoch":800,"state":"active+clean","last_fresh":"2024-09-21T10:04:00.637701+0000","last_change":"2024-09-21T10:03:58.587829+0000","last_active":"2024-09-21T10:04:00.637701+0000","last_peered":"2024-09-21T10:04:00.637701+0000","last_clean":"2024-09-21T10:04:00.637701+0000","last_became_active":"2024-09-21T09:58:42.592444+0000","last_became_peered":"2024-09-21T09:58:42.592444+0000","last_unstale":"2024-09-21T10:04:00.637701+0000","last_undegraded":"2024-09-21T10:04:00.637701+0000","last_fullsized":"2024-09-21T10:04:00.637701+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":390,"log_dups_size":0,"ondisk_log_size":390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:21:49.091262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00054663800000000001,"stat_sum":{"num_bytes":7200768,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":96,"num_read_kb":118579,"num_write":170,"num_write_kb":30456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":9854976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[{"start":"1b8","length":"2"}]},{"pgid":"2.9","version":"0'0","reported_seq":1215,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.654833+0000","last_change":"2024-09-21T09:59:50.929490+0000","last_active":"2024-09-21T10:04:02.654833+0000","last_peered":"2024-09-21T10:04:02.654833+0000","last_clean":"2024-09-21T10:04:02.654833+0000","last_became_active":"2024-09-21T09:59:50.929148+0000","last_became_peered":"2024-09-21T09:59:50.929148+0000","last_unstale":"2024-09-21T10:04:02.654833+0000","last_undegraded":"2024-09-21T10:04:02.654833+0000","last_fullsized":"2024-09-21T10:04:02.654833+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T15:06:51.477965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,3],"acting":[1,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1137,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.882641+0000","last_change":"2024-09-21T09:59:50.924568+0000","last_active":"2024-09-21T10:04:01.882641+0000","last_peered":"2024-09-21T10:04:01.882641+0000","last_clean":"2024-09-21T10:04:01.882641+0000","last_became_active":"2024-09-21T09:59:50.924131+0000","last_became_peered":"2024-09-21T09:59:50.924131+0000","last_unstale":"2024-09-21T10:04:01.882641+0000","last_undegraded":"2024-09-21T10:04:01.882641+0000","last_fullsized":"2024-09-21T10:04:01.882641+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:34:55.044498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.d","version":"802'641","reported_seq":3675,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.672210+0000","last_change":"2024-09-21T10:04:02.674128+0000","last_active":"2024-09-21T10:04:02.672210+0000","last_peered":"2024-09-21T10:04:02.672210+0000","last_clean":"2024-09-21T10:04:02.672210+0000","last_became_active":"2024-09-21T09:59:50.930195+0000","last_became_peered":"2024-09-21T09:59:50.930195+0000","last_unstale":"2024-09-21T10:04:02.672210+0000","last_undegraded":"2024-09-21T10:04:02.672210+0000","last_fullsized":"2024-09-21T10:04:02.672210+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":641,"log_dups_size":0,"ondisk_log_size":641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T12:40:29.736091+0000","scrub_duration":0,"objects_trimmed":3,"snaptrim_duration":0.020475912999999998,"stat_sum":{"num_bytes":6963200,"num_objects":2,"num_object_clones":1,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":127,"num_read_kb":96442,"num_write":458,"num_write_kb":58904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":7831552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"2.a","version":"0'0","reported_seq":1215,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.656493+0000","last_change":"2024-09-21T09:59:50.916296+0000","last_active":"2024-09-21T10:04:02.656493+0000","last_peered":"2024-09-21T10:04:02.656493+0000","last_clean":"2024-09-21T10:04:02.656493+0000","last_became_active":"2024-09-21T09:59:50.915991+0000","last_became_peered":"2024-09-21T09:59:50.915991+0000","last_unstale":"2024-09-21T10:04:02.656493+0000","last_undegraded":"2024-09-21T10:04:02.656493+0000","last_fullsized":"2024-09-21T10:04:02.656493+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:00:49.037494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.b","version":"0'0","reported_seq":1138,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.883152+0000","last_change":"2024-09-21T09:58:08.510081+0000","last_active":"2024-09-21T10:04:01.883152+0000","last_peered":"2024-09-21T10:04:01.883152+0000","last_clean":"2024-09-21T10:04:01.883152+0000","last_became_active":"2024-09-21T09:58:08.509709+0000","last_became_peered":"2024-09-21T09:58:08.509709+0000","last_unstale":"2024-09-21T10:04:01.883152+0000","last_undegraded":"2024-09-21T10:04:01.883152+0000","last_fullsized":"2024-09-21T10:04:01.883152+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:35:17.899403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.c","version":"802'1027","reported_seq":4187,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.666591+0000","last_change":"2024-09-21T10:04:02.667555+0000","last_active":"2024-09-21T10:04:02.666591+0000","last_peered":"2024-09-21T10:04:02.666591+0000","last_clean":"2024-09-21T10:04:02.666591+0000","last_became_active":"2024-09-21T09:58:08.508818+0000","last_became_peered":"2024-09-21T09:58:08.508818+0000","last_unstale":"2024-09-21T10:04:02.666591+0000","last_undegraded":"2024-09-21T10:04:02.666591+0000","last_fullsized":"2024-09-21T10:04:02.666591+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":70,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:51:23.682690+0000","last_clean_scrub_stamp":"2024-09-21T09:51:23.682690+0000","objects_scrubbed":0,"log_size":1027,"log_dups_size":0,"ondisk_log_size":1027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T10:50:28.152885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.010118794,"stat_sum":{"num_bytes":30523392,"num_objects":4,"num_object_clones":2,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":1,"num_read":284,"num_read_kb":610832,"num_write":484,"num_write_kb":129936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":18956288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"1b2","length":"1"},{"start":"1b8","length":"2"}]},{"pgid":"2.b","version":"581'3","reported_seq":1015,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.661111+0000","last_change":"2024-09-21T09:59:50.927739+0000","last_active":"2024-09-21T10:04:02.661111+0000","last_peered":"2024-09-21T10:04:02.661111+0000","last_clean":"2024-09-21T10:04:02.661111+0000","last_became_active":"2024-09-21T09:59:50.926898+0000","last_became_peered":"2024-09-21T09:59:50.926898+0000","last_unstale":"2024-09-21T10:04:02.661111+0000","last_undegraded":"2024-09-21T10:04:02.661111+0000","last_fullsized":"2024-09-21T10:04:02.661111+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T19:28:13.254792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,5],"acting":[7,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1039,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.655795+0000","last_change":"2024-09-21T09:59:15.571327+0000","last_active":"2024-09-21T10:04:02.655795+0000","last_peered":"2024-09-21T10:04:02.655795+0000","last_clean":"2024-09-21T10:04:02.655795+0000","last_became_active":"2024-09-21T09:59:15.570685+0000","last_became_peered":"2024-09-21T09:59:15.570685+0000","last_unstale":"2024-09-21T10:04:02.655795+0000","last_undegraded":"2024-09-21T10:04:02.655795+0000","last_fullsized":"2024-09-21T10:04:02.655795+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:34:08.214635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1d","version":"0'0","reported_seq":1063,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651518+0000","last_change":"2024-09-21T09:59:15.568823+0000","last_active":"2024-09-21T10:04:02.651518+0000","last_peered":"2024-09-21T10:04:02.651518+0000","last_clean":"2024-09-21T10:04:02.651518+0000","last_became_active":"2024-09-21T09:59:15.568555+0000","last_became_peered":"2024-09-21T09:59:15.568555+0000","last_unstale":"2024-09-21T10:04:02.651518+0000","last_undegraded":"2024-09-21T10:04:02.651518+0000","last_fullsized":"2024-09-21T10:04:02.651518+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:52:51.462320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1c","version":"57'2","reported_seq":1015,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.661791+0000","last_change":"2024-09-21T09:59:50.925825+0000","last_active":"2024-09-21T10:04:02.661791+0000","last_peered":"2024-09-21T10:04:02.661791+0000","last_clean":"2024-09-21T10:04:02.661791+0000","last_became_active":"2024-09-21T09:59:50.925528+0000","last_became_peered":"2024-09-21T09:59:50.925528+0000","last_unstale":"2024-09-21T10:04:02.661791+0000","last_undegraded":"2024-09-21T10:04:02.661791+0000","last_fullsized":"2024-09-21T10:04:02.661791+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T17:53:16.461449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"0'0","reported_seq":1163,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.653170+0000","last_change":"2024-09-21T09:58:08.510958+0000","last_active":"2024-09-21T10:04:02.653170+0000","last_peered":"2024-09-21T10:04:02.653170+0000","last_clean":"2024-09-21T10:04:02.653170+0000","last_became_active":"2024-09-21T09:58:08.510668+0000","last_became_peered":"2024-09-21T09:58:08.510668+0000","last_unstale":"2024-09-21T10:04:02.653170+0000","last_undegraded":"2024-09-21T10:04:02.653170+0000","last_fullsized":"2024-09-21T10:04:02.653170+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T14:20:33.276931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1d","version":"57'2","reported_seq":1004,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.662059+0000","last_change":"2024-09-21T09:59:50.923907+0000","last_active":"2024-09-21T10:04:02.662059+0000","last_peered":"2024-09-21T10:04:02.662059+0000","last_clean":"2024-09-21T10:04:02.662059+0000","last_became_active":"2024-09-21T09:59:50.923356+0000","last_became_peered":"2024-09-21T09:59:50.923356+0000","last_unstale":"2024-09-21T10:04:02.662059+0000","last_undegraded":"2024-09-21T10:04:02.662059+0000","last_fullsized":"2024-09-21T10:04:02.662059+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T13:10:38.614093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"0'0","reported_seq":1239,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651597+0000","last_change":"2024-09-21T09:58:42.603420+0000","last_active":"2024-09-21T10:04:02.651597+0000","last_peered":"2024-09-21T10:04:02.651597+0000","last_clean":"2024-09-21T10:04:02.651597+0000","last_became_active":"2024-09-21T09:58:42.603099+0000","last_became_peered":"2024-09-21T09:58:42.603099+0000","last_unstale":"2024-09-21T10:04:02.651597+0000","last_undegraded":"2024-09-21T10:04:02.651597+0000","last_fullsized":"2024-09-21T10:04:02.651597+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:29:16.597217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1e","version":"581'2","reported_seq":1149,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.883159+0000","last_change":"2024-09-21T09:58:42.609013+0000","last_active":"2024-09-21T10:04:01.883159+0000","last_peered":"2024-09-21T10:04:01.883159+0000","last_clean":"2024-09-21T10:04:01.883159+0000","last_became_active":"2024-09-21T09:58:42.608640+0000","last_became_peered":"2024-09-21T09:58:42.608640+0000","last_unstale":"2024-09-21T10:04:01.883159+0000","last_undegraded":"2024-09-21T10:04:01.883159+0000","last_fullsized":"2024-09-21T10:04:01.883159+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T20:37:20.908016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"0'0","reported_seq":1129,"reported_epoch":801,"state":"active+clean","last_fresh":"2024-09-21T10:04:01.883032+0000","last_change":"2024-09-21T09:59:15.576990+0000","last_active":"2024-09-21T10:04:01.883032+0000","last_peered":"2024-09-21T10:04:01.883032+0000","last_clean":"2024-09-21T10:04:01.883032+0000","last_became_active":"2024-09-21T09:59:15.576648+0000","last_became_peered":"2024-09-21T09:59:15.576648+0000","last_unstale":"2024-09-21T10:04:01.883032+0000","last_undegraded":"2024-09-21T10:04:01.883032+0000","last_fullsized":"2024-09-21T10:04:01.883032+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":54,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:23.027120+0000","last_clean_scrub_stamp":"2024-09-21T09:47:23.027120+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T16:08:19.024663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.1f","version":"581'5","reported_seq":1251,"reported_epoch":802,"state":"active+clean","last_fresh":"2024-09-21T10:04:02.651861+0000","last_change":"2024-09-21T09:59:50.923810+0000","last_active":"2024-09-21T10:04:02.651861+0000","last_peered":"2024-09-21T10:04:02.651861+0000","last_clean":"2024-09-21T10:04:02.651861+0000","last_became_active":"2024-09-21T09:59:50.923439+0000","last_became_peered":"2024-09-21T09:59:50.923439+0000","last_unstale":"2024-09-21T10:04:02.651861+0000","last_undegraded":"2024-09-21T10:04:02.651861+0000","last_fullsized":"2024-09-21T10:04:02.651861+0000","mapping_epoch":560,"log_start":"0'0","ondisk_log_start":"0'0","created":51,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-21T09:47:19.840262+0000","last_clean_scrub_stamp":"2024-09-21T09:47:19.840262+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-22T21:14:54.687163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":218267648,"num_objects":63,"num_object_clones":22,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":6,"num_read":3805,"num_read_kb":5656364,"num_write":8774,"num_write_kb":1894968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":368304128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":337162240,"data_stored":444461056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17042,"ondisk_log_size":17042,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":64,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":192,"acting":192,"num_store_stats":8},{"poolid":2,"num_pg":32,"stat_sum":{"num_bytes":6602,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":79,"num_read_kb":86,"num_write":68,"num_write_kb":35,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":19806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44709,"internal_metadata":0},"log_size":68,"ondisk_log_size":68,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":840,"num_read_kb":1709,"num_write":269,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":160,"ondisk_log_size":160,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":6,"up_from":525,"seq":2254857830459,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":140920,"kb_used_data":41324,"kb_used_omap":35,"kb_used_meta":99548,"kb_avail":93604232,"statfs":{"total":95995035648,"available":95850733568,"internally_reserved":0,"allocated":42315776,"data_stored":49756412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36846,"internal_metadata":101937170},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":426,"seq":1829656068175,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":108928,"kb_used_data":32172,"kb_used_omap":18,"kb_used_meta":76717,"kb_avail":93636224,"statfs":{"total":95995035648,"available":95883493376,"internally_reserved":0,"allocated":32944128,"data_stored":33610424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18865,"internal_metadata":78558799},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":393,"seq":1687922147414,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":177216,"kb_used_data":38516,"kb_used_omap":21,"kb_used_meta":138666,"kb_avail":93567936,"statfs":{"total":95995035648,"available":95813566464,"internally_reserved":0,"allocated":39440384,"data_stored":48249289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22030,"internal_metadata":141994482},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":325,"seq":1395864371300,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":426088,"kb_used_data":69396,"kb_used_omap":30,"kb_used_meta":356641,"kb_avail":93319064,"statfs":{"total":95995035648,"available":95558721536,"internally_reserved":0,"allocated":71061504,"data_stored":78933386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30861,"internal_metadata":365201267},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":7,"up_from":560,"seq":2405181685812,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":151688,"kb_used_data":35640,"kb_used_omap":32,"kb_used_meta":115999,"kb_avail":93593464,"statfs":{"total":95995035648,"available":95839707136,"internally_reserved":0,"allocated":36495360,"data_stored":36175401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33598,"internal_metadata":118783170},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":494,"seq":2121713844290,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302644,"kb_used_data":62248,"kb_used_omap":48,"kb_used_meta":240335,"kb_avail":93442508,"statfs":{"total":95995035648,"available":95685128192,"internally_reserved":0,"allocated":63741952,"data_stored":74907563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":50091,"internal_metadata":246103125},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":460,"seq":1975684956232,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":263692,"kb_used_data":65088,"kb_used_omap":37,"kb_used_meta":198554,"kb_avail":93481460,"statfs":{"total":95995035648,"available":95725015040,"internally_reserved":0,"allocated":66650112,"data_stored":89920288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":38298,"internal_metadata":203319910},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":359,"seq":1541893259357,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":406424,"kb_used_data":73288,"kb_used_omap":30,"kb_used_meta":333089,"kb_avail":93338728,"statfs":{"total":95995035648,"available":95578857472,"internally_reserved":0,"allocated":75046912,"data_stored":82922124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31313,"internal_metadata":341083567},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6298,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4478,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4613,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6176,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6378,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5717,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":5816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5948,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5101,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":58322944,"data_stored":71266304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":62902272,"data_stored":75845632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":27308032,"data_stored":41172992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":20807680,"data_stored":26537984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":62885888,"data_stored":91217920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":51589120,"data_stored":67825664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":29585408,"data_stored":42086400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":23760896,"data_stored":28508160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-21T10:04:08.529 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-21T10:04:08.529 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-21T10:04:08.794 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:04:08.907 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:08 smithi033 ceph-mon[98546]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:08.907 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:08 smithi033 ceph-mon[98546]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T10:04:08.907 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:08 smithi033 ceph-mon[96568]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:08.907 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:08 smithi033 ceph-mon[96568]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T10:04:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:08 smithi105 ceph-mon[86649]: from='client.44513 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-21T10:04:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:08 smithi105 ceph-mon[86649]: osdmap e803: 8 total, 8 up, 8 in 2024-09-21T10:04:09.992 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:09 smithi033 ceph-mon[98546]: pgmap v842: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-21T10:04:09.993 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:09 smithi033 ceph-mon[96568]: pgmap v842: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-21T10:04:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:09 smithi105 ceph-mon[86649]: pgmap v842: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.5 MiB/s wr, 6 op/s 2024-09-21T10:04:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:04:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:04:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:10.682 INFO:teuthology.orchestra.run.smithi033.stderr:pool 'unique_pool_0' removed 2024-09-21T10:04:10.986 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:10 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3273840018' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:10.987 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:10 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:10.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:10 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3273840018' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:10.987 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:10 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:10 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3273840018' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:11.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:10 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:04:11.299 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-21T10:04:11.299 INFO:teuthology.task.print:**** done end ec-rados-default.yaml 2024-09-21T10:04:11.300 INFO:teuthology.task.full_sequential:In full_sequential, running task print... 2024-09-21T10:04:11.300 INFO:teuthology.task.print:**** done start rados_api.yaml 2024-09-21T10:04:11.300 INFO:teuthology.task.full_sequential:In full_sequential, running task workunit... 2024-09-21T10:04:11.300 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-21T10:04:11.300 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-21T10:04:11.301 DEBUG:teuthology.orchestra.run.smithi033:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-21T10:04:11.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:04:11.319 INFO:teuthology.orchestra.run.smithi033.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-21T10:04:11.319 DEBUG:teuthology.orchestra.run.smithi033:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-21T10:04:11.377 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-21T10:04:11.377 DEBUG:teuthology.orchestra.run.smithi033:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-21T10:04:11.435 INFO:tasks.workunit:timeout=3h 2024-09-21T10:04:11.435 INFO:tasks.workunit:cleanup=True 2024-09-21T10:04:11.435 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-21T10:04:11.494 INFO:tasks.workunit.client.0.smithi033.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-21T10:04:11.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:04:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:04:11] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.51.0" 2024-09-21T10:04:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:11 smithi033 ceph-mon[98546]: pgmap v844: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.0 MiB/s wr, 3 op/s 2024-09-21T10:04:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:11 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:04:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:11 smithi033 ceph-mon[98546]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T10:04:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:11 smithi033 ceph-mon[96568]: pgmap v844: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.0 MiB/s wr, 3 op/s 2024-09-21T10:04:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:11 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:04:11.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:11 smithi033 ceph-mon[96568]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T10:04:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:11 smithi105 ceph-mon[86649]: pgmap v844: 121 pgs: 121 active+clean; 206 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 2.0 MiB/s wr, 3 op/s 2024-09-21T10:04:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:11 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:04:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:11 smithi105 ceph-mon[86649]: osdmap e804: 8 total, 8 up, 8 in 2024-09-21T10:04:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:13 smithi033 ceph-mon[96568]: pgmap v846: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:04:13.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:13 smithi033 ceph-mon[98546]: pgmap v846: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:04:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:13 smithi105 ceph-mon[86649]: pgmap v846: 105 pgs: 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:04:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:15 smithi033 ceph-mon[98546]: pgmap v847: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:04:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:15 smithi033 ceph-mon[96568]: pgmap v847: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:04:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:15 smithi105 ceph-mon[86649]: pgmap v847: 105 pgs: 105 active+clean; 583 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:04:16.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:16.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:17.779 DEBUG:teuthology.orchestra.run.smithi033:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-21T10:04:17.835 INFO:tasks.workunit.client.0.smithi033.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-21T10:04:17.838 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T10:04:17.838 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-21T10:04:17.889 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-21T10:04:17.941 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-21T10:04:17.980 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-21T10:04:17.982 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T10:04:17.982 INFO:tasks.workunit.client.0.smithi033.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-21T10:04:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:17 smithi033 ceph-mon[98546]: pgmap v848: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:17 smithi033 ceph-mon[96568]: pgmap v848: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:18.022 INFO:tasks.workunit.client.0.smithi033.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-21T10:04:18.025 DEBUG:teuthology.orchestra.run.smithi033:> set -ex 2024-09-21T10:04:18.025 DEBUG:teuthology.orchestra.run.smithi033:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-21T10:04:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:17 smithi105 ceph-mon[86649]: pgmap v848: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:18.082 INFO:tasks.workunit:Running workunits matching cls on client.0... 2024-09-21T10:04:18.083 INFO:tasks.workunit:Running workunit cls/test_cls_2pc_queue.sh... 2024-09-21T10:04:18.083 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_2pc_queue.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 2024-09-21T10:04:18.149 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:04:18.149 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 22 tests from 1 test suite. 2024-09-21T10:04:18.149 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:04:18.149 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 22 tests from TestCls2PCQueue 2024-09-21T10:04:18.149 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.GetCapacity 2024-09-21T10:04:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:19 smithi033 ceph-mon[98546]: pgmap v849: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:19 smithi033 ceph-mon[98546]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T10:04:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:19 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/895830943' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:19 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:19 smithi033 ceph-mon[96568]: pgmap v849: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:19 smithi033 ceph-mon[96568]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T10:04:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:19 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/895830943' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:19.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:19 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:19 smithi105 ceph-mon[86649]: pgmap v849: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:19 smithi105 ceph-mon[86649]: osdmap e805: 8 total, 8 up, 8 in 2024-09-21T10:04:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:19 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/895830943' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:19 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:04:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:04:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:20.720 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.GetCapacity (2571 ms) 2024-09-21T10:04:20.721 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.AsyncGetCapacity 2024-09-21T10:04:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:20 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:20.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:20 smithi033 ceph-mon[98546]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T10:04:20.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:20 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:20.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:20 smithi033 ceph-mon[96568]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T10:04:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:20 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:21.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:20 smithi105 ceph-mon[86649]: osdmap e806: 8 total, 8 up, 8 in 2024-09-21T10:04:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:21 smithi033 ceph-mon[98546]: pgmap v852: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:21 smithi033 ceph-mon[98546]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T10:04:21.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:21 smithi033 ceph-mon[96568]: pgmap v852: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:21.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:21 smithi033 ceph-mon[96568]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T10:04:21.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:04:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:04:21] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.51.0" 2024-09-21T10:04:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:21 smithi105 ceph-mon[86649]: pgmap v852: 137 pgs: 9 creating+peering, 23 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:21 smithi105 ceph-mon[86649]: osdmap e807: 8 total, 8 up, 8 in 2024-09-21T10:04:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:22 smithi033 ceph-mon[98546]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T10:04:22.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:22 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1722695791' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:22 smithi033 ceph-mon[96568]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T10:04:22.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:22 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1722695791' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:22 smithi105 ceph-mon[86649]: osdmap e808: 8 total, 8 up, 8 in 2024-09-21T10:04:23.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:22 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1722695791' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:23.738 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.AsyncGetCapacity (3018 ms) 2024-09-21T10:04:23.738 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.Reserve 2024-09-21T10:04:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:23 smithi033 ceph-mon[98546]: pgmap v855: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:23 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:23 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1722695791' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:23 smithi033 ceph-mon[98546]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T10:04:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:23 smithi033 ceph-mon[96568]: pgmap v855: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:23 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:23 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1722695791' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:23.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:23 smithi033 ceph-mon[96568]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T10:04:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:23 smithi105 ceph-mon[86649]: pgmap v855: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:23 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:23 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1722695791' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:23 smithi105 ceph-mon[86649]: osdmap e809: 8 total, 8 up, 8 in 2024-09-21T10:04:24.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:24 smithi033 ceph-mon[98546]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T10:04:24.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:24 smithi033 ceph-mon[96568]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T10:04:25.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:24 smithi105 ceph-mon[86649]: osdmap e810: 8 total, 8 up, 8 in 2024-09-21T10:04:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[98546]: pgmap v858: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:04:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[98546]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T10:04:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/717842494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[96568]: pgmap v858: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:04:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[96568]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T10:04:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/717842494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:25.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:25 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:25 smithi105 ceph-mon[86649]: pgmap v858: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:25 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:25 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:04:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:25 smithi105 ceph-mon[86649]: osdmap e811: 8 total, 8 up, 8 in 2024-09-21T10:04:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:25 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/717842494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:26.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:25 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:26.764 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.Reserve (3025 ms) 2024-09-21T10:04:26.764 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.AsyncReserve 2024-09-21T10:04:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:26 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:26 smithi105 ceph-mon[86649]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T10:04:27.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:26 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:27.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:26 smithi033 ceph-mon[98546]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T10:04:27.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:26 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:27.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:26 smithi033 ceph-mon[96568]: osdmap e812: 8 total, 8 up, 8 in 2024-09-21T10:04:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:27 smithi105 ceph-mon[86649]: pgmap v861: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:27 smithi105 ceph-mon[86649]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T10:04:28.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:27 smithi033 ceph-mon[98546]: pgmap v861: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:28.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:27 smithi033 ceph-mon[98546]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T10:04:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:27 smithi033 ceph-mon[96568]: pgmap v861: 137 pgs: 12 creating+peering, 20 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:27 smithi033 ceph-mon[96568]: osdmap e813: 8 total, 8 up, 8 in 2024-09-21T10:04:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:28 smithi105 ceph-mon[86649]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T10:04:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:28 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/799228230' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:28 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:28 smithi033 ceph-mon[98546]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T10:04:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:28 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/799228230' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:28 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:29.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:28 smithi033 ceph-mon[96568]: osdmap e814: 8 total, 8 up, 8 in 2024-09-21T10:04:29.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:28 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/799228230' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:29.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:28 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:29.788 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.AsyncReserve (3025 ms) 2024-09-21T10:04:29.788 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.Commit 2024-09-21T10:04:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:29 smithi105 ceph-mon[86649]: pgmap v864: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:29 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:29 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:29 smithi105 ceph-mon[86649]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T10:04:30.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:29 smithi033 ceph-mon[98546]: pgmap v864: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:30.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:29 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:30.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:29 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:30.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:29 smithi033 ceph-mon[98546]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T10:04:30.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:29 smithi033 ceph-mon[96568]: pgmap v864: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:30.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:29 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:30.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:29 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:30.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:29 smithi033 ceph-mon[96568]: osdmap e815: 8 total, 8 up, 8 in 2024-09-21T10:04:30.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:04:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:04:30.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:30 smithi033 ceph-mon[96568]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:31.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:30 smithi033 ceph-mon[98546]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T10:04:31.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:30 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:31.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:30 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:31.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:30 smithi105 ceph-mon[86649]: osdmap e816: 8 total, 8 up, 8 in 2024-09-21T10:04:31.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:30 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:31.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:30 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:04:31] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.51.0" 2024-09-21T10:04:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[98546]: pgmap v867: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:04:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[98546]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T10:04:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3943733296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[96568]: pgmap v867: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:04:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[96568]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T10:04:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3943733296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:31 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:32.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:31 smithi105 ceph-mon[86649]: pgmap v867: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:32.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:31 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:32.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:31 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:04:32.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:31 smithi105 ceph-mon[86649]: osdmap e817: 8 total, 8 up, 8 in 2024-09-21T10:04:32.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:31 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3943733296' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:32.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:31 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:32.813 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.Commit (3025 ms) 2024-09-21T10:04:32.814 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.Abort 2024-09-21T10:04:33.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:32 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:33.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:32 smithi033 ceph-mon[98546]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T10:04:33.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:32 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:33.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:32 smithi033 ceph-mon[96568]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T10:04:33.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:32 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:33.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:32 smithi105 ceph-mon[86649]: osdmap e818: 8 total, 8 up, 8 in 2024-09-21T10:04:34.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:33 smithi033 ceph-mon[98546]: pgmap v870: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:34.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:33 smithi033 ceph-mon[98546]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T10:04:34.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:33 smithi033 ceph-mon[96568]: pgmap v870: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:34.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:33 smithi033 ceph-mon[96568]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T10:04:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:33 smithi105 ceph-mon[86649]: pgmap v870: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:33 smithi105 ceph-mon[86649]: osdmap e819: 8 total, 8 up, 8 in 2024-09-21T10:04:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:34 smithi033 ceph-mon[98546]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T10:04:35.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:34 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2327466201' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:35.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:34 smithi033 ceph-mon[96568]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T10:04:35.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:34 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2327466201' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:34 smithi105 ceph-mon[86649]: osdmap e820: 8 total, 8 up, 8 in 2024-09-21T10:04:35.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:34 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2327466201' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:35.840 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.Abort (3027 ms) 2024-09-21T10:04:35.840 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.ReserveError 2024-09-21T10:04:36.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:35 smithi033 ceph-mon[98546]: pgmap v873: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:36.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:35 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:36.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:35 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2327466201' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:36.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:35 smithi033 ceph-mon[98546]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T10:04:36.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:35 smithi033 ceph-mon[96568]: pgmap v873: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:36.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:35 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:36.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:35 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2327466201' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:36.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:35 smithi033 ceph-mon[96568]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T10:04:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:35 smithi105 ceph-mon[86649]: pgmap v873: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:35 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:35 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2327466201' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:35 smithi105 ceph-mon[86649]: osdmap e821: 8 total, 8 up, 8 in 2024-09-21T10:04:37.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:36 smithi033 ceph-mon[98546]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T10:04:37.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:36 smithi033 ceph-mon[96568]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T10:04:37.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:36 smithi105 ceph-mon[86649]: osdmap e822: 8 total, 8 up, 8 in 2024-09-21T10:04:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[98546]: pgmap v876: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:04:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[98546]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T10:04:38.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3966376655' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[96568]: pgmap v876: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:04:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[96568]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T10:04:38.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:37 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3966376655' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:37 smithi105 ceph-mon[86649]: pgmap v876: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:37 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:37 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:04:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:37 smithi105 ceph-mon[86649]: osdmap e823: 8 total, 8 up, 8 in 2024-09-21T10:04:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:37 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3966376655' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:38.858 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.ReserveError (3018 ms) 2024-09-21T10:04:38.858 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.CommitError 2024-09-21T10:04:39.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:38 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3966376655' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:39.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:38 smithi033 ceph-mon[98546]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T10:04:39.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:38 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3966376655' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:39.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:38 smithi033 ceph-mon[96568]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T10:04:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:38 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3966376655' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:38 smithi105 ceph-mon[86649]: osdmap e824: 8 total, 8 up, 8 in 2024-09-21T10:04:40.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:39 smithi105 ceph-mon[86649]: pgmap v879: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:40.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:39 smithi105 ceph-mon[86649]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T10:04:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:39 smithi033 ceph-mon[98546]: pgmap v879: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:39 smithi033 ceph-mon[98546]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T10:04:40.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:39 smithi033 ceph-mon[96568]: pgmap v879: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:40.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:39 smithi033 ceph-mon[96568]: osdmap e825: 8 total, 8 up, 8 in 2024-09-21T10:04:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:04:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:04:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:40 smithi033 ceph-mon[98546]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T10:04:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:40 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1313431834' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:41.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:40 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:41.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:40 smithi033 ceph-mon[96568]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T10:04:41.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:40 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1313431834' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:41.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:40 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:40 smithi105 ceph-mon[86649]: osdmap e826: 8 total, 8 up, 8 in 2024-09-21T10:04:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:40 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1313431834' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:41.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:40 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:41.872 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.CommitError (3014 ms) 2024-09-21T10:04:41.872 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.AbortError 2024-09-21T10:04:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-mon[98546]: pgmap v882: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-mon[98546]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T10:04:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-mon[96568]: pgmap v882: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-mon[96568]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T10:04:41.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:04:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:04:41] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.51.0" 2024-09-21T10:04:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:41 smithi105 ceph-mon[86649]: pgmap v882: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:41 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:41 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:41 smithi105 ceph-mon[86649]: osdmap e827: 8 total, 8 up, 8 in 2024-09-21T10:04:43.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:42 smithi033 ceph-mon[98546]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T10:04:43.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:42 smithi033 ceph-mon[96568]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T10:04:43.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:42 smithi105 ceph-mon[86649]: osdmap e828: 8 total, 8 up, 8 in 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[96568]: pgmap v885: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[96568]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/765140513' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[98546]: pgmap v885: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[98546]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T10:04:44.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/765140513' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:44.149 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:43 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:43 smithi105 ceph-mon[86649]: pgmap v885: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:43 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:43 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:04:44.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:43 smithi105 ceph-mon[86649]: osdmap e829: 8 total, 8 up, 8 in 2024-09-21T10:04:44.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:43 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/765140513' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:44.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:43 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:44.897 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.AbortError (3024 ms) 2024-09-21T10:04:44.897 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.MultiReserve 2024-09-21T10:04:45.163 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:44 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:45.163 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:44 smithi033 ceph-mon[98546]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T10:04:45.163 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:44 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:45.163 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:44 smithi033 ceph-mon[96568]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T10:04:45.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:44 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:45.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:44 smithi105 ceph-mon[86649]: osdmap e830: 8 total, 8 up, 8 in 2024-09-21T10:04:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:45 smithi033 ceph-mon[98546]: pgmap v888: 137 pgs: 7 creating+peering, 22 unknown, 108 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:45 smithi033 ceph-mon[98546]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T10:04:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:45 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:45 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:45 smithi033 ceph-mon[96568]: pgmap v888: 137 pgs: 7 creating+peering, 22 unknown, 108 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:45 smithi033 ceph-mon[96568]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T10:04:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:45 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:45 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:45 smithi105 ceph-mon[86649]: pgmap v888: 137 pgs: 7 creating+peering, 22 unknown, 108 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:45 smithi105 ceph-mon[86649]: osdmap e831: 8 total, 8 up, 8 in 2024-09-21T10:04:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:45 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:45 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:04:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:46 smithi033 ceph-mon[98546]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T10:04:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:46 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2272771516' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:46 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:47.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:46 smithi033 ceph-mon[96568]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T10:04:47.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:46 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2272771516' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:47.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:46 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:47.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:46 smithi105 ceph-mon[86649]: osdmap e832: 8 total, 8 up, 8 in 2024-09-21T10:04:47.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:46 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2272771516' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:47.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:46 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:47.917 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.MultiReserve (3021 ms) 2024-09-21T10:04:47.917 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.MultiCommit 2024-09-21T10:04:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:47 smithi033 ceph-mon[98546]: pgmap v891: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:47 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:47 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:47 smithi033 ceph-mon[98546]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T10:04:48.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:47 smithi033 ceph-mon[96568]: pgmap v891: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:47 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:47 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:47 smithi033 ceph-mon[96568]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T10:04:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:47 smithi105 ceph-mon[86649]: pgmap v891: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:47 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:47 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:47 smithi105 ceph-mon[86649]: osdmap e833: 8 total, 8 up, 8 in 2024-09-21T10:04:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:48 smithi033 ceph-mon[98546]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T10:04:49.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:48 smithi033 ceph-mon[96568]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T10:04:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:48 smithi105 ceph-mon[86649]: osdmap e834: 8 total, 8 up, 8 in 2024-09-21T10:04:50.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:49 smithi105 ceph-mon[86649]: pgmap v894: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:50.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:49 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:50.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:49 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:04:50.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:49 smithi105 ceph-mon[86649]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T10:04:50.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:49 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3838824096' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[96568]: pgmap v894: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:04:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[96568]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T10:04:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3838824096' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:50.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[98546]: pgmap v894: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:04:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[98546]: osdmap e835: 8 total, 8 up, 8 in 2024-09-21T10:04:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:49 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3838824096' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:04:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:04:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:04:50.938 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.MultiCommit (3020 ms) 2024-09-21T10:04:50.938 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.MultiAbort 2024-09-21T10:04:51.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:50 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3838824096' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:51.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:50 smithi033 ceph-mon[98546]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T10:04:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:50 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3838824096' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:50 smithi033 ceph-mon[96568]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T10:04:51.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:50 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3838824096' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:51.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:50 smithi105 ceph-mon[86649]: osdmap e836: 8 total, 8 up, 8 in 2024-09-21T10:04:51.926 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:04:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:04:51] "GET /metrics HTTP/1.1" 200 34086 "" "Prometheus/2.51.0" 2024-09-21T10:04:51.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:51 smithi033 ceph-mon[96568]: pgmap v897: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:51.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:51 smithi033 ceph-mon[96568]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T10:04:51.927 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:51 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:52.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:51 smithi033 ceph-mon[98546]: pgmap v897: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:52.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:51 smithi033 ceph-mon[98546]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T10:04:52.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:51 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:51 smithi105 ceph-mon[86649]: pgmap v897: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:51 smithi105 ceph-mon[86649]: osdmap e837: 8 total, 8 up, 8 in 2024-09-21T10:04:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:51 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:04:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[98546]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T10:04:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2380004472' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:04:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:04:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[96568]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T10:04:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2380004472' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:04:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:04:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:52 smithi105 ceph-mon[86649]: osdmap e838: 8 total, 8 up, 8 in 2024-09-21T10:04:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:52 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2380004472' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:52 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:04:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:04:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:04:53.963 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.MultiAbort (3024 ms) 2024-09-21T10:04:53.963 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.ReserveCommit 2024-09-21T10:04:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:53 smithi033 ceph-mon[98546]: pgmap v900: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:53 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:53 smithi033 ceph-mon[98546]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T10:04:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:53 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:53 smithi033 ceph-mon[96568]: pgmap v900: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:53 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:53 smithi033 ceph-mon[96568]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T10:04:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:53 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:53 smithi105 ceph-mon[86649]: pgmap v900: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:53 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:53 smithi105 ceph-mon[86649]: osdmap e839: 8 total, 8 up, 8 in 2024-09-21T10:04:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:53 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:04:55.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:54 smithi033 ceph-mon[98546]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T10:04:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:54 smithi033 ceph-mon[96568]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T10:04:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:54 smithi105 ceph-mon[86649]: osdmap e840: 8 total, 8 up, 8 in 2024-09-21T10:04:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[98546]: pgmap v903: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:04:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[98546]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T10:04:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2658123575' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:56.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[96568]: pgmap v903: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:04:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[96568]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T10:04:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2658123575' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:55 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:55 smithi105 ceph-mon[86649]: pgmap v903: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:55 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:04:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:55 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:04:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:55 smithi105 ceph-mon[86649]: osdmap e841: 8 total, 8 up, 8 in 2024-09-21T10:04:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:55 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2658123575' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:55 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:56.984 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.ReserveCommit (3023 ms) 2024-09-21T10:04:56.984 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.ReserveAbort 2024-09-21T10:04:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:56 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:56 smithi033 ceph-mon[98546]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T10:04:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:56 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:56 smithi033 ceph-mon[96568]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T10:04:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:56 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:04:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:56 smithi105 ceph-mon[86649]: osdmap e842: 8 total, 8 up, 8 in 2024-09-21T10:04:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:57 smithi033 ceph-mon[98546]: pgmap v906: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:57 smithi033 ceph-mon[98546]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T10:04:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:57 smithi033 ceph-mon[96568]: pgmap v906: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:57 smithi033 ceph-mon[96568]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T10:04:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:57 smithi105 ceph-mon[86649]: pgmap v906: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:04:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:57 smithi105 ceph-mon[86649]: osdmap e843: 8 total, 8 up, 8 in 2024-09-21T10:04:59.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:58 smithi033 ceph-mon[96568]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T10:04:59.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:58 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2427192056' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:59.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:58 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:59.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:58 smithi033 ceph-mon[98546]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T10:04:59.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:58 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2427192056' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:59.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:58 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:58 smithi105 ceph-mon[86649]: osdmap e844: 8 total, 8 up, 8 in 2024-09-21T10:04:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:58 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2427192056' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:04:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:58 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:00.011 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.ReserveAbort (3027 ms) 2024-09-21T10:05:00.011 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.ManualCleanup 2024-09-21T10:05:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:59 smithi033 ceph-mon[98546]: pgmap v909: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:59 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:59 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:04:59 smithi033 ceph-mon[98546]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T10:05:00.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:59 smithi033 ceph-mon[96568]: pgmap v909: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:59 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:59 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:04:59 smithi033 ceph-mon[96568]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T10:05:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:59 smithi105 ceph-mon[86649]: pgmap v909: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:59 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:59 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:04:59 smithi105 ceph-mon[86649]: osdmap e845: 8 total, 8 up, 8 in 2024-09-21T10:05:00.281 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:05:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:05:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:01 smithi105 ceph-mon[86649]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T10:05:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:01.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:01 smithi033 ceph-mon[98546]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T10:05:01.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:01.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:01.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:01 smithi033 ceph-mon[96568]: osdmap e846: 8 total, 8 up, 8 in 2024-09-21T10:05:01.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:01.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:05:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:05:01] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.51.0" 2024-09-21T10:05:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:02 smithi105 ceph-mon[86649]: pgmap v912: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:02 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:02 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:05:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:02 smithi105 ceph-mon[86649]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T10:05:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:02 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/610660992' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[96568]: pgmap v912: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:05:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[96568]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T10:05:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/610660992' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:02.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[98546]: pgmap v912: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:02.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:02.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:05:02.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[98546]: osdmap e847: 8 total, 8 up, 8 in 2024-09-21T10:05:02.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:02 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/610660992' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:03.031 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.ManualCleanup (3019 ms) 2024-09-21T10:05:03.031 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.Cleanup 2024-09-21T10:05:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:03 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/610660992' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:03 smithi105 ceph-mon[86649]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T10:05:03.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:03 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/610660992' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:03.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:03 smithi033 ceph-mon[98546]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T10:05:03.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:03 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/610660992' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:03.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:03 smithi033 ceph-mon[96568]: osdmap e848: 8 total, 8 up, 8 in 2024-09-21T10:05:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:04 smithi105 ceph-mon[86649]: pgmap v915: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:04 smithi105 ceph-mon[86649]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T10:05:04.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:04 smithi033 ceph-mon[98546]: pgmap v915: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:04.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:04 smithi033 ceph-mon[98546]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T10:05:04.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:04 smithi033 ceph-mon[96568]: pgmap v915: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:04.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:04 smithi033 ceph-mon[96568]: osdmap e849: 8 total, 8 up, 8 in 2024-09-21T10:05:05.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:05 smithi033 ceph-mon[98546]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T10:05:05.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:05 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3472200539' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:05.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:05 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:05.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:05 smithi033 ceph-mon[98546]: pgmap v918: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:05.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:05 smithi033 ceph-mon[96568]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T10:05:05.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:05 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3472200539' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:05.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:05 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:05.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:05 smithi033 ceph-mon[96568]: pgmap v918: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:05.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:05 smithi105 ceph-mon[86649]: osdmap e850: 8 total, 8 up, 8 in 2024-09-21T10:05:05.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:05 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3472200539' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:05.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:05 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:05.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:05 smithi105 ceph-mon[86649]: pgmap v918: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:06.050 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.Cleanup (3019 ms) 2024-09-21T10:05:06.050 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.MultiProducer 2024-09-21T10:05:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:06 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:06 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:06 smithi033 ceph-mon[96568]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T10:05:06.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:06 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:06.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:06 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:06.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:06 smithi033 ceph-mon[98546]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T10:05:06.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:06 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:06.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:06 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:06.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:06 smithi105 ceph-mon[86649]: osdmap e851: 8 total, 8 up, 8 in 2024-09-21T10:05:07.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:07 smithi033 ceph-mon[96568]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T10:05:07.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:07 smithi033 ceph-mon[96568]: pgmap v921: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:07.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:07 smithi033 ceph-mon[98546]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T10:05:07.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:07 smithi033 ceph-mon[98546]: pgmap v921: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:07.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:07 smithi105 ceph-mon[86649]: osdmap e852: 8 total, 8 up, 8 in 2024-09-21T10:05:07.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:07 smithi105 ceph-mon[86649]: pgmap v921: 105 pgs: 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:08.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:08.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:05:08.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[96568]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T10:05:08.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1957560190' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:08.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:08.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:08.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:05:08.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[98546]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T10:05:08.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1957560190' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:08.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:08 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:08.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:08 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:08.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:08 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:05:08.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:08 smithi105 ceph-mon[86649]: osdmap e853: 8 total, 8 up, 8 in 2024-09-21T10:05:08.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:08 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1957560190' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:08.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:08 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:09.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:09 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:09.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:09 smithi033 ceph-mon[98546]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T10:05:09.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:09 smithi033 ceph-mon[98546]: pgmap v924: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:09.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:09 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:09.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:09 smithi033 ceph-mon[96568]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T10:05:09.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:09 smithi033 ceph-mon[96568]: pgmap v924: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:09.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:09 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:09.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:09 smithi105 ceph-mon[86649]: osdmap e854: 8 total, 8 up, 8 in 2024-09-21T10:05:09.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:09 smithi105 ceph-mon[86649]: pgmap v924: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-09-21T10:05:10.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:10 smithi033 ceph-mon[98546]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T10:05:10.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:10 smithi033 ceph-mon[96568]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T10:05:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:10 smithi105 ceph-mon[86649]: osdmap e855: 8 total, 8 up, 8 in 2024-09-21T10:05:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:05:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:05:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:11.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:11 smithi033 ceph-mon[98546]: pgmap v926: 137 pgs: 3 creating+peering, 11 unknown, 123 active+clean; 1.0 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 4.4 MiB/s wr, 4.98k op/s 2024-09-21T10:05:11.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:11 smithi033 ceph-mon[96568]: pgmap v926: 137 pgs: 3 creating+peering, 11 unknown, 123 active+clean; 1.0 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 4.4 MiB/s wr, 4.98k op/s 2024-09-21T10:05:11.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:11 smithi105 ceph-mon[86649]: pgmap v926: 137 pgs: 3 creating+peering, 11 unknown, 123 active+clean; 1.0 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 8.2 MiB/s rd, 4.4 MiB/s wr, 4.98k op/s 2024-09-21T10:05:11.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:05:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:05:11] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.51.0" 2024-09-21T10:05:13.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:13 smithi105 ceph-mon[86649]: pgmap v927: 137 pgs: 3 creating+peering, 134 active+clean; 1.0 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.3 MiB/s wr, 3.68k op/s 2024-09-21T10:05:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:13 smithi033 ceph-mon[98546]: pgmap v927: 137 pgs: 3 creating+peering, 134 active+clean; 1.0 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.3 MiB/s wr, 3.68k op/s 2024-09-21T10:05:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:13 smithi033 ceph-mon[96568]: pgmap v927: 137 pgs: 3 creating+peering, 134 active+clean; 1.0 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 3.3 MiB/s wr, 3.68k op/s 2024-09-21T10:05:15.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:15 smithi105 ceph-mon[86649]: pgmap v928: 137 pgs: 137 active+clean; 1.0 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.6 MiB/s wr, 2.98k op/s 2024-09-21T10:05:15.797 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:15 smithi033 ceph-mon[98546]: pgmap v928: 137 pgs: 137 active+clean; 1.0 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.6 MiB/s wr, 2.98k op/s 2024-09-21T10:05:15.798 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:15 smithi033 ceph-mon[96568]: pgmap v928: 137 pgs: 137 active+clean; 1.0 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 2.6 MiB/s wr, 2.98k op/s 2024-09-21T10:05:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:17.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:17.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:17.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:17.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:17 smithi105 ceph-mon[86649]: pgmap v929: 137 pgs: 137 active+clean; 2.1 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s rd, 7.6 MiB/s wr, 8.58k op/s 2024-09-21T10:05:18.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:17 smithi033 ceph-mon[98546]: pgmap v929: 137 pgs: 137 active+clean; 2.1 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s rd, 7.6 MiB/s wr, 8.58k op/s 2024-09-21T10:05:18.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:17 smithi033 ceph-mon[96568]: pgmap v929: 137 pgs: 137 active+clean; 2.1 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 41 MiB/s rd, 7.6 MiB/s wr, 8.58k op/s 2024-09-21T10:05:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:19 smithi105 ceph-mon[86649]: pgmap v930: 137 pgs: 137 active+clean; 2.1 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 6.4 MiB/s wr, 7.21k op/s 2024-09-21T10:05:20.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:19 smithi033 ceph-mon[98546]: pgmap v930: 137 pgs: 137 active+clean; 2.1 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 6.4 MiB/s wr, 7.21k op/s 2024-09-21T10:05:20.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:19 smithi033 ceph-mon[96568]: pgmap v930: 137 pgs: 137 active+clean; 2.1 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 6.4 MiB/s wr, 7.21k op/s 2024-09-21T10:05:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:05:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:05:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:20.793 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.MultiProducer (14744 ms) 2024-09-21T10:05:20.793 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.AsyncConsumer 2024-09-21T10:05:21.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:05:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:05:21] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.51.0" 2024-09-21T10:05:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:21 smithi033 ceph-mon[98546]: pgmap v931: 137 pgs: 137 active+clean; 2.3 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 206 MiB/s rd, 6.5 MiB/s wr, 7.88k op/s 2024-09-21T10:05:21.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:21 smithi033 ceph-mon[98546]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T10:05:21.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:21 smithi033 ceph-mon[96568]: pgmap v931: 137 pgs: 137 active+clean; 2.3 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 206 MiB/s rd, 6.5 MiB/s wr, 7.88k op/s 2024-09-21T10:05:21.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:21 smithi033 ceph-mon[96568]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T10:05:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:21 smithi105 ceph-mon[86649]: pgmap v931: 137 pgs: 137 active+clean; 2.3 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 206 MiB/s rd, 6.5 MiB/s wr, 7.88k op/s 2024-09-21T10:05:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:21 smithi105 ceph-mon[86649]: osdmap e856: 8 total, 8 up, 8 in 2024-09-21T10:05:23.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:22 smithi033 ceph-mon[98546]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T10:05:23.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:22 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/4256550738' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:23.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:22 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:22 smithi033 ceph-mon[96568]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T10:05:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:22 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/4256550738' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:23.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:22 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:22 smithi105 ceph-mon[86649]: osdmap e857: 8 total, 8 up, 8 in 2024-09-21T10:05:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:22 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/4256550738' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:23.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:22 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:23 smithi033 ceph-mon[98546]: pgmap v934: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:23 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:23 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:24.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:23 smithi033 ceph-mon[98546]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T10:05:24.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:23 smithi033 ceph-mon[96568]: pgmap v934: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:23 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:23 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:24.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:23 smithi033 ceph-mon[96568]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T10:05:24.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:23 smithi105 ceph-mon[86649]: pgmap v934: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:24.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:23 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:24.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:23 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:24.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:23 smithi105 ceph-mon[86649]: osdmap e858: 8 total, 8 up, 8 in 2024-09-21T10:05:24.830 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.AsyncConsumer (4037 ms) 2024-09-21T10:05:24.830 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.MultiProducerConsumer 2024-09-21T10:05:25.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:24 smithi033 ceph-mon[96568]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T10:05:25.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:24 smithi033 ceph-mon[98546]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T10:05:25.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:24 smithi105 ceph-mon[86649]: osdmap e859: 8 total, 8 up, 8 in 2024-09-21T10:05:26.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:25 smithi033 ceph-mon[96568]: pgmap v937: 137 pgs: 8 creating+peering, 19 unknown, 110 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:26.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:25 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:26.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:25 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:05:26.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:25 smithi033 ceph-mon[96568]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T10:05:26.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:25 smithi033 ceph-mon[98546]: pgmap v937: 137 pgs: 8 creating+peering, 19 unknown, 110 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:26.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:25 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:26.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:25 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:05:26.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:25 smithi033 ceph-mon[98546]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T10:05:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:25 smithi105 ceph-mon[86649]: pgmap v937: 137 pgs: 8 creating+peering, 19 unknown, 110 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:25 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:25 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:05:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:25 smithi105 ceph-mon[86649]: osdmap e860: 8 total, 8 up, 8 in 2024-09-21T10:05:27.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:26 smithi033 ceph-mon[98546]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T10:05:27.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:26 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/4021260239' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:27.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:26 smithi033 ceph-mon[96568]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T10:05:27.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:26 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/4021260239' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:27.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:26 smithi105 ceph-mon[86649]: osdmap e861: 8 total, 8 up, 8 in 2024-09-21T10:05:27.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:26 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/4021260239' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:28.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:27 smithi033 ceph-mon[98546]: pgmap v940: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:28.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:27 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:28.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:27 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/4021260239' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:28.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:27 smithi033 ceph-mon[98546]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T10:05:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:27 smithi033 ceph-mon[96568]: pgmap v940: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:28.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:27 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:27 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/4021260239' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:28.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:27 smithi033 ceph-mon[96568]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T10:05:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:27 smithi105 ceph-mon[86649]: pgmap v940: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:27 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:27 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/4021260239' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:28.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:27 smithi105 ceph-mon[86649]: osdmap e862: 8 total, 8 up, 8 in 2024-09-21T10:05:29.181 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:28 smithi105 ceph-mon[86649]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T10:05:29.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:28 smithi033 ceph-mon[98546]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T10:05:29.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:28 smithi033 ceph-mon[96568]: osdmap e863: 8 total, 8 up, 8 in 2024-09-21T10:05:30.192 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:29 smithi105 ceph-mon[86649]: pgmap v943: 137 pgs: 2 creating+peering, 26 unknown, 109 active+clean; 782 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.9 MiB/s wr, 2.25k op/s 2024-09-21T10:05:30.192 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:29 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:30.192 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:29 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:05:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:29 smithi033 ceph-mon[98546]: pgmap v943: 137 pgs: 2 creating+peering, 26 unknown, 109 active+clean; 782 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.9 MiB/s wr, 2.25k op/s 2024-09-21T10:05:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:29 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:30.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:29 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:05:30.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:29 smithi033 ceph-mon[96568]: pgmap v943: 137 pgs: 2 creating+peering, 26 unknown, 109 active+clean; 782 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.8 MiB/s rd, 1.9 MiB/s wr, 2.25k op/s 2024-09-21T10:05:30.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:29 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:30.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:29 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:05:30.460 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:05:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:05:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:31.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:30 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:31.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:30 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:31.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:30 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:31 smithi033 ceph-mon[98546]: pgmap v944: 137 pgs: 2 creating+peering, 7 unknown, 128 active+clean; 782 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.4 MiB/s wr, 1.59k op/s 2024-09-21T10:05:31.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:31 smithi033 ceph-mon[96568]: pgmap v944: 137 pgs: 2 creating+peering, 7 unknown, 128 active+clean; 782 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.4 MiB/s wr, 1.59k op/s 2024-09-21T10:05:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:05:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:05:31] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.51.0" 2024-09-21T10:05:32.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:31 smithi105 ceph-mon[86649]: pgmap v944: 137 pgs: 2 creating+peering, 7 unknown, 128 active+clean; 782 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 1.4 MiB/s wr, 1.59k op/s 2024-09-21T10:05:34.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:33 smithi033 ceph-mon[98546]: pgmap v945: 137 pgs: 137 active+clean; 782 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.2 MiB/s wr, 1.35k op/s 2024-09-21T10:05:34.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:33 smithi033 ceph-mon[96568]: pgmap v945: 137 pgs: 137 active+clean; 782 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.2 MiB/s wr, 1.35k op/s 2024-09-21T10:05:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:33 smithi105 ceph-mon[86649]: pgmap v945: 137 pgs: 137 active+clean; 782 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 1.2 MiB/s wr, 1.35k op/s 2024-09-21T10:05:36.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:35 smithi033 ceph-mon[98546]: pgmap v946: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 5.4 MiB/s wr, 6.46k op/s 2024-09-21T10:05:36.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:35 smithi033 ceph-mon[96568]: pgmap v946: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 5.4 MiB/s wr, 6.46k op/s 2024-09-21T10:05:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:35 smithi105 ceph-mon[86649]: pgmap v946: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 5.4 MiB/s wr, 6.46k op/s 2024-09-21T10:05:38.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:37 smithi033 ceph-mon[98546]: pgmap v947: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 4.5 MiB/s wr, 5.36k op/s 2024-09-21T10:05:38.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:37 smithi033 ceph-mon[96568]: pgmap v947: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 4.5 MiB/s wr, 5.36k op/s 2024-09-21T10:05:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:37 smithi105 ceph-mon[86649]: pgmap v947: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 4.5 MiB/s wr, 5.36k op/s 2024-09-21T10:05:40.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:39 smithi105 ceph-mon[86649]: pgmap v948: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 MiB/s rd, 5.0 MiB/s wr, 6.06k op/s 2024-09-21T10:05:40.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:39 smithi033 ceph-mon[98546]: pgmap v948: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 MiB/s rd, 5.0 MiB/s wr, 6.06k op/s 2024-09-21T10:05:40.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:39 smithi033 ceph-mon[96568]: pgmap v948: 137 pgs: 137 active+clean; 1.6 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 73 MiB/s rd, 5.0 MiB/s wr, 6.06k op/s 2024-09-21T10:05:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:05:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:05:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:41 smithi033 ceph-mon[98546]: pgmap v949: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.37k op/s 2024-09-21T10:05:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:05:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:05:41] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.51.0" 2024-09-21T10:05:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:41 smithi033 ceph-mon[96568]: pgmap v949: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.37k op/s 2024-09-21T10:05:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:41 smithi105 ceph-mon[86649]: pgmap v949: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.37k op/s 2024-09-21T10:05:44.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:43 smithi033 ceph-mon[98546]: pgmap v950: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.37k op/s 2024-09-21T10:05:44.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:43 smithi033 ceph-mon[96568]: pgmap v950: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.37k op/s 2024-09-21T10:05:44.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:43 smithi105 ceph-mon[86649]: pgmap v950: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 4.4 MiB/s wr, 5.37k op/s 2024-09-21T10:05:44.885 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.MultiProducerConsumer (20052 ms) 2024-09-21T10:05:44.885 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.ReserveSpillover 2024-09-21T10:05:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:45 smithi033 ceph-mon[98546]: pgmap v951: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 103 MiB/s rd, 5.6 MiB/s wr, 6.87k op/s 2024-09-21T10:05:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:45 smithi033 ceph-mon[98546]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T10:05:46.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:45 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:45 smithi033 ceph-mon[96568]: pgmap v951: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 103 MiB/s rd, 5.6 MiB/s wr, 6.87k op/s 2024-09-21T10:05:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:45 smithi033 ceph-mon[96568]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T10:05:46.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:45 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:45 smithi105 ceph-mon[86649]: pgmap v951: 137 pgs: 137 active+clean; 1.6 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 103 MiB/s rd, 5.6 MiB/s wr, 6.87k op/s 2024-09-21T10:05:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:45 smithi105 ceph-mon[86649]: osdmap e864: 8 total, 8 up, 8 in 2024-09-21T10:05:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:45 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:05:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:46 smithi033 ceph-mon[98546]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T10:05:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:46 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3095802003' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:47.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:46 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:47.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:46 smithi033 ceph-mon[96568]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T10:05:47.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:46 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3095802003' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:47.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:46 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:47.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:46 smithi105 ceph-mon[86649]: osdmap e865: 8 total, 8 up, 8 in 2024-09-21T10:05:47.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:46 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3095802003' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:47.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:46 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:47 smithi033 ceph-mon[98546]: pgmap v954: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:47 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:47 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:47 smithi033 ceph-mon[98546]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T10:05:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:47 smithi033 ceph-mon[96568]: pgmap v954: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:47 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:47 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:47 smithi033 ceph-mon[96568]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T10:05:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:47 smithi105 ceph-mon[86649]: pgmap v954: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:47 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:47 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:47 smithi105 ceph-mon[86649]: osdmap e866: 8 total, 8 up, 8 in 2024-09-21T10:05:48.915 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.ReserveSpillover (4033 ms) 2024-09-21T10:05:48.916 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.CommitSpillover 2024-09-21T10:05:49.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:48 smithi033 ceph-mon[98546]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T10:05:49.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:48 smithi033 ceph-mon[96568]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T10:05:49.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:48 smithi105 ceph-mon[86649]: osdmap e867: 8 total, 8 up, 8 in 2024-09-21T10:05:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:49 smithi105 ceph-mon[86649]: pgmap v957: 137 pgs: 5 creating+peering, 22 unknown, 110 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:49 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:49 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:05:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:49 smithi105 ceph-mon[86649]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T10:05:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:49 smithi033 ceph-mon[96568]: pgmap v957: 137 pgs: 5 creating+peering, 22 unknown, 110 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:49 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:49 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:05:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:49 smithi033 ceph-mon[96568]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T10:05:50.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:49 smithi033 ceph-mon[98546]: pgmap v957: 137 pgs: 5 creating+peering, 22 unknown, 110 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:50.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:49 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:49 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:05:50.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:49 smithi033 ceph-mon[98546]: osdmap e868: 8 total, 8 up, 8 in 2024-09-21T10:05:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:05:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:05:50.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:05:51.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:50 smithi033 ceph-mon[98546]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T10:05:51.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:50 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2516494236' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:51.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:50 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:51.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:50 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:51.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:50 smithi033 ceph-mon[96568]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T10:05:51.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:50 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2516494236' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:51.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:50 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:51.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:50 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:51.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:50 smithi105 ceph-mon[86649]: osdmap e869: 8 total, 8 up, 8 in 2024-09-21T10:05:51.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:50 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2516494236' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:51.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:50 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:51.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:50 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:51.923 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:51 smithi033 ceph-mon[96568]: pgmap v960: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:51.923 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:51 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:51.923 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:05:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:05:51] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.51.0" 2024-09-21T10:05:52.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:51 smithi033 ceph-mon[96568]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T10:05:52.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:51 smithi033 ceph-mon[98546]: pgmap v960: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:52.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:51 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:52.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:51 smithi033 ceph-mon[98546]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T10:05:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:51 smithi105 ceph-mon[86649]: pgmap v960: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:51 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:51 smithi105 ceph-mon[86649]: osdmap e870: 8 total, 8 up, 8 in 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[98546]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[96568]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:53.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:52 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:52 smithi105 ceph-mon[86649]: osdmap e871: 8 total, 8 up, 8 in 2024-09-21T10:05:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:05:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:05:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:05:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:52 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:05:54.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:53 smithi033 ceph-mon[96568]: pgmap v963: 137 pgs: 4 creating+peering, 23 unknown, 110 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:54.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:53 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:54.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:53 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:05:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:53 smithi033 ceph-mon[98546]: pgmap v963: 137 pgs: 4 creating+peering, 23 unknown, 110 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:53 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:54.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:53 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:05:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:53 smithi105 ceph-mon[86649]: pgmap v963: 137 pgs: 4 creating+peering, 23 unknown, 110 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:53 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:05:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:53 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:05:54.961 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.CommitSpillover (6046 ms) 2024-09-21T10:05:54.962 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestCls2PCQueue.AbortSpillover 2024-09-21T10:05:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:55 smithi033 ceph-mon[98546]: pgmap v964: 137 pgs: 4 creating+peering, 7 unknown, 126 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:55 smithi033 ceph-mon[98546]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T10:05:56.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:55 smithi033 ceph-mon[96568]: pgmap v964: 137 pgs: 4 creating+peering, 7 unknown, 126 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:55 smithi033 ceph-mon[96568]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T10:05:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:55 smithi105 ceph-mon[86649]: pgmap v964: 137 pgs: 4 creating+peering, 7 unknown, 126 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:55 smithi105 ceph-mon[86649]: osdmap e872: 8 total, 8 up, 8 in 2024-09-21T10:05:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:56 smithi033 ceph-mon[98546]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T10:05:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:56 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2147771958' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:56 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:56 smithi033 ceph-mon[96568]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T10:05:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:56 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2147771958' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:56 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:56 smithi105 ceph-mon[86649]: osdmap e873: 8 total, 8 up, 8 in 2024-09-21T10:05:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:56 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2147771958' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:56 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:05:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:57 smithi033 ceph-mon[96568]: pgmap v967: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:57 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:57 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:58.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:57 smithi033 ceph-mon[96568]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T10:05:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:57 smithi033 ceph-mon[98546]: pgmap v967: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:57 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:57 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:57 smithi033 ceph-mon[98546]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T10:05:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:57 smithi105 ceph-mon[86649]: pgmap v967: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:05:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:57 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:05:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:57 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-128413-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:05:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:57 smithi105 ceph-mon[86649]: osdmap e874: 8 total, 8 up, 8 in 2024-09-21T10:05:59.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:58 smithi033 ceph-mon[98546]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T10:05:59.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:58 smithi033 ceph-mon[96568]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T10:05:59.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:58 smithi105 ceph-mon[86649]: osdmap e875: 8 total, 8 up, 8 in 2024-09-21T10:05:59.990 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestCls2PCQueue.AbortSpillover (5029 ms) 2024-09-21T10:05:59.990 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 22 tests from TestCls2PCQueue (101841 ms total) 2024-09-21T10:05:59.990 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:05:59.990 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:05:59.990 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 22 tests from 1 test suite ran. (101841 ms total) 2024-09-21T10:05:59.990 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 22 tests. 2024-09-21T10:05:59.992 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T10:05:59.992 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:06:00.020 INFO:tasks.workunit:Running workunit cls/test_cls_cas.sh... 2024-09-21T10:06:00.021 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_cas.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cas.sh 2024-09-21T10:06:00.088 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:06:00.088 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 6 tests from 2 test suites. 2024-09-21T10:06:00.088 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:06:00.088 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 5 tests from cls_cas 2024-09-21T10:06:00.089 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_cas.get_put 2024-09-21T10:06:00.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:59 smithi033 ceph-mon[96568]: pgmap v970: 137 pgs: 3 creating+peering, 25 unknown, 109 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:00.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:59 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:00.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:05:59 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:00.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:59 smithi033 ceph-mon[98546]: pgmap v970: 137 pgs: 3 creating+peering, 25 unknown, 109 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:00.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:59 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:00.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:05:59 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:59 smithi105 ceph-mon[86649]: pgmap v970: 137 pgs: 3 creating+peering, 25 unknown, 109 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:59 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:00.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:05:59 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:00.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:06:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:06:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:01.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:00 smithi033 ceph-mon[98546]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T10:06:01.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:00 smithi033 ceph-mon[96568]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T10:06:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:00 smithi105 ceph-mon[86649]: osdmap e876: 8 total, 8 up, 8 in 2024-09-21T10:06:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:06:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:06:01] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.51.0" 2024-09-21T10:06:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:02 smithi105 ceph-mon[86649]: pgmap v972: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:02 smithi105 ceph-mon[86649]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T10:06:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:02 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/367020758' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:02.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:01 smithi033 ceph-mon[98546]: pgmap v972: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:02.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:01 smithi033 ceph-mon[98546]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T10:06:02.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:01 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/367020758' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:01 smithi033 ceph-mon[96568]: pgmap v972: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:01 smithi033 ceph-mon[96568]: osdmap e877: 8 total, 8 up, 8 in 2024-09-21T10:06:02.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:01 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/367020758' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:03.018 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_cas.get_put (2929 ms) 2024-09-21T10:06:03.018 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_cas.wrong_put 2024-09-21T10:06:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:03 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/367020758' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:03 smithi105 ceph-mon[86649]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T10:06:03.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:03 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/367020758' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:03.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:03 smithi033 ceph-mon[98546]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T10:06:03.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:03 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/367020758' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:03.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:03 smithi033 ceph-mon[96568]: osdmap e878: 8 total, 8 up, 8 in 2024-09-21T10:06:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:04 smithi105 ceph-mon[86649]: pgmap v975: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:04 smithi105 ceph-mon[86649]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T10:06:04.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:04 smithi033 ceph-mon[98546]: pgmap v975: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:04.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:04 smithi033 ceph-mon[98546]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T10:06:04.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:04 smithi033 ceph-mon[96568]: pgmap v975: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:04.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:04 smithi033 ceph-mon[96568]: osdmap e879: 8 total, 8 up, 8 in 2024-09-21T10:06:05.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:05 smithi105 ceph-mon[86649]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T10:06:05.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:05 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/746480561' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:05.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:05 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:05.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:05 smithi033 ceph-mon[98546]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T10:06:05.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:05 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/746480561' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:05.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:05 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:05.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:05 smithi033 ceph-mon[96568]: osdmap e880: 8 total, 8 up, 8 in 2024-09-21T10:06:05.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:05 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/746480561' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:05.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:05 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:06.037 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_cas.wrong_put (3020 ms) 2024-09-21T10:06:06.037 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_cas.dup_get 2024-09-21T10:06:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:06 smithi105 ceph-mon[86649]: pgmap v978: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:06 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:06 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:06 smithi105 ceph-mon[86649]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T10:06:06.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:06 smithi033 ceph-mon[98546]: pgmap v978: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:06.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:06 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:06.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:06 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:06.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:06 smithi033 ceph-mon[98546]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T10:06:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:06 smithi033 ceph-mon[96568]: pgmap v978: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:06 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:06.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:06 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:06.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:06 smithi033 ceph-mon[96568]: osdmap e881: 8 total, 8 up, 8 in 2024-09-21T10:06:07.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:07 smithi033 ceph-mon[98546]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T10:06:07.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:07 smithi033 ceph-mon[96568]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T10:06:07.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:07 smithi105 ceph-mon[86649]: osdmap e882: 8 total, 8 up, 8 in 2024-09-21T10:06:08.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[98546]: pgmap v981: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:08.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:08.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:08.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[98546]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T10:06:08.948 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2520321130' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[96568]: pgmap v981: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[96568]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T10:06:08.949 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:08 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2520321130' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:08.949 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:08 smithi105 ceph-mon[86649]: pgmap v981: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:08.949 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:08 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:08.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:08 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:08.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:08 smithi105 ceph-mon[86649]: osdmap e883: 8 total, 8 up, 8 in 2024-09-21T10:06:08.950 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:08 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2520321130' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:09.056 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_cas.dup_get (3019 ms) 2024-09-21T10:06:09.056 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_cas.dup_put 2024-09-21T10:06:09.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:09 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2520321130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:09.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:09 smithi033 ceph-mon[96568]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T10:06:09.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:09 smithi033 ceph-mon[96568]: pgmap v984: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:09.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:09 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2520321130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:09.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:09 smithi033 ceph-mon[98546]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T10:06:09.497 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:09 smithi033 ceph-mon[98546]: pgmap v984: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:09.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:09 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2520321130' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:09.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:09 smithi105 ceph-mon[86649]: osdmap e884: 8 total, 8 up, 8 in 2024-09-21T10:06:09.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:09 smithi105 ceph-mon[86649]: pgmap v984: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:10.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:10 smithi033 ceph-mon[96568]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T10:06:10.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:10 smithi033 ceph-mon[98546]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T10:06:10.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:10 smithi105 ceph-mon[86649]: osdmap e885: 8 total, 8 up, 8 in 2024-09-21T10:06:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:06:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:06:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:11.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-mon[98546]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T10:06:11.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3428539475' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:11.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:11.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-mon[98546]: pgmap v987: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:11.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-mon[96568]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T10:06:11.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3428539475' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:11.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:11.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-mon[96568]: pgmap v987: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:11.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:11 smithi105 ceph-mon[86649]: osdmap e886: 8 total, 8 up, 8 in 2024-09-21T10:06:11.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:11 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3428539475' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:11.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:11 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:11.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:11 smithi105 ceph-mon[86649]: pgmap v987: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:11.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:06:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:06:11] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.51.0" 2024-09-21T10:06:12.079 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_cas.dup_put (3023 ms) 2024-09-21T10:06:12.080 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_cas.get_wrong_data 2024-09-21T10:06:12.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:12 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:12.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:12 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:12.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:12 smithi033 ceph-mon[98546]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T10:06:12.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:12 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:12.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:12 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:12.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:12 smithi033 ceph-mon[96568]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T10:06:12.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:12 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:12.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:12 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:12.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:12 smithi105 ceph-mon[86649]: osdmap e887: 8 total, 8 up, 8 in 2024-09-21T10:06:13.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:13 smithi033 ceph-mon[98546]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T10:06:13.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:13 smithi033 ceph-mon[98546]: pgmap v990: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:13.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:13 smithi033 ceph-mon[96568]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T10:06:13.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:13 smithi033 ceph-mon[96568]: pgmap v990: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:13.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:13 smithi105 ceph-mon[86649]: osdmap e888: 8 total, 8 up, 8 in 2024-09-21T10:06:13.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:13 smithi105 ceph-mon[86649]: pgmap v990: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:14.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:14 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:14.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:14 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:14.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:14 smithi033 ceph-mon[98546]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T10:06:14.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:14 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3542261892' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:14.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:14 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:14.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:14 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:14.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:14 smithi033 ceph-mon[96568]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T10:06:14.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:14 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3542261892' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:14.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:14 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:14.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:14 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:14.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:14 smithi105 ceph-mon[86649]: osdmap e889: 8 total, 8 up, 8 in 2024-09-21T10:06:14.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:14 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3542261892' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_cas.get_wrong_data (3027 ms) 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 5 tests from cls_cas (15018 ms total) 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 1 test from chunk_refs_t 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] chunk_refs_t.size 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:1 60 by_object 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:2 103 by_object 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:4 189 by_object 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:8 361 by_object 2024-09-21T10:06:15.107 INFO:tasks.workunit.client.0.smithi033.stdout:16 112 by_hash(32 bits) 2024-09-21T10:06:15.108 INFO:tasks.workunit.client.0.smithi033.stdout:32 208 by_hash(32 bits) 2024-09-21T10:06:15.108 INFO:tasks.workunit.client.0.smithi033.stdout:64 400 by_hash(32 bits) 2024-09-21T10:06:15.108 INFO:tasks.workunit.client.0.smithi033.stdout:128 509 by_hash(10 bits) 2024-09-21T10:06:15.108 INFO:tasks.workunit.client.0.smithi033.stdout:256 499 by_hash(7 bits) 2024-09-21T10:06:15.108 INFO:tasks.workunit.client.0.smithi033.stdout:512 462 by_hash(5 bits) 2024-09-21T10:06:15.108 INFO:tasks.workunit.client.0.smithi033.stdout:1024 466 by_hash(5 bits) 2024-09-21T10:06:15.109 INFO:tasks.workunit.client.0.smithi033.stdout:2048 466 by_hash(5 bits) 2024-09-21T10:06:15.109 INFO:tasks.workunit.client.0.smithi033.stdout:4096 470 by_hash(5 bits) 2024-09-21T10:06:15.109 INFO:tasks.workunit.client.0.smithi033.stdout:8192 498 by_hash(4 bits) 2024-09-21T10:06:15.110 INFO:tasks.workunit.client.0.smithi033.stdout:16384 278 by_hash(3 bits) 2024-09-21T10:06:15.112 INFO:tasks.workunit.client.0.smithi033.stdout:32768 337 by_hash(2 bits) 2024-09-21T10:06:15.117 INFO:tasks.workunit.client.0.smithi033.stdout:65536 367 by_hash(2 bits) 2024-09-21T10:06:15.124 INFO:tasks.workunit.client.0.smithi033.stdout:131072 387 by_hash(1 bits) 2024-09-21T10:06:15.139 INFO:tasks.workunit.client.0.smithi033.stdout:262144 331 by_pool 2024-09-21T10:06:15.169 INFO:tasks.workunit.client.0.smithi033.stdout:524288 21 count 2024-09-21T10:06:15.221 INFO:tasks.workunit.client.0.smithi033.stdout:1048576 21 count 2024-09-21T10:06:15.332 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] chunk_refs_t.size (226 ms) 2024-09-21T10:06:15.332 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 1 test from chunk_refs_t (226 ms total) 2024-09-21T10:06:15.332 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:06:15.332 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:06:15.332 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 6 tests from 2 test suites ran. (15244 ms total) 2024-09-21T10:06:15.332 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 6 tests. 2024-09-21T10:06:15.334 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T10:06:15.334 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:06:15.356 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:15 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3542261892' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:15.356 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:15 smithi033 ceph-mon[98546]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T10:06:15.356 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:15 smithi033 ceph-mon[98546]: pgmap v993: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:15.356 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:15 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3542261892' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:15.357 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:15 smithi033 ceph-mon[96568]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T10:06:15.357 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:15 smithi033 ceph-mon[96568]: pgmap v993: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:15.360 INFO:tasks.workunit:Running workunit cls/test_cls_cmpomap.sh... 2024-09-21T10:06:15.360 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_cmpomap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_cmpomap.sh 2024-09-21T10:06:15.428 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:06:15.428 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 31 tests from 1 test suite. 2024-09-21T10:06:15.428 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:06:15.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:15 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3542261892' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129101-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:15.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:15 smithi105 ceph-mon[86649]: osdmap e890: 8 total, 8 up, 8 in 2024-09-21T10:06:15.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:15 smithi105 ceph-mon[86649]: pgmap v993: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:16.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:16 smithi033 ceph-mon[98546]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T10:06:16.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:16.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:16 smithi033 ceph-mon[96568]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T10:06:16.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:16.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:16 smithi105 ceph-mon[86649]: osdmap e891: 8 total, 8 up, 8 in 2024-09-21T10:06:16.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:17.124 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 31 tests from CmpOmap 2024-09-21T10:06:17.124 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str 2024-09-21T10:06:17.137 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str (13 ms) 2024-09-21T10:06:17.137 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_str_default 2024-09-21T10:06:17.146 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_noexist_str_default (9 ms) 2024-09-21T10:06:17.146 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64 2024-09-21T10:06:17.149 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64 (3 ms) 2024-09-21T10:06:17.149 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_noexist_u64_default 2024-09-21T10:06:17.153 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_noexist_u64_default (4 ms) 2024-09-21T10:06:17.153 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_str 2024-09-21T10:06:17.158 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_str (5 ms) 2024-09-21T10:06:17.158 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_u64 2024-09-21T10:06:17.166 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_u64 (8 ms) 2024-09-21T10:06:17.166 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_input 2024-09-21T10:06:17.170 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_input (4 ms) 2024-09-21T10:06:17.170 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_default 2024-09-21T10:06:17.172 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_default (3 ms) 2024-09-21T10:06:17.173 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_u64_empty_default 2024-09-21T10:06:17.175 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_u64_empty_default (2 ms) 2024-09-21T10:06:17.175 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_u64_invalid_value 2024-09-21T10:06:17.179 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_u64_invalid_value (4 ms) 2024-09-21T10:06:17.179 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_at_max_keys 2024-09-21T10:06:17.180 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_at_max_keys (1 ms) 2024-09-21T10:06:17.180 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_vals_over_max_keys 2024-09-21T10:06:17.180 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_vals_over_max_keys (0 ms) 2024-09-21T10:06:17.180 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str 2024-09-21T10:06:17.187 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str (7 ms) 2024-09-21T10:06:17.187 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_str_default 2024-09-21T10:06:17.193 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_str_default (6 ms) 2024-09-21T10:06:17.194 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64 2024-09-21T10:06:17.200 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64 (7 ms) 2024-09-21T10:06:17.200 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_noexist_u64_default 2024-09-21T10:06:17.207 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_noexist_u64_default (7 ms) 2024-09-21T10:06:17.207 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_str 2024-09-21T10:06:17.214 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_str (7 ms) 2024-09-21T10:06:17.214 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64 2024-09-21T10:06:17.221 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_u64 (7 ms) 2024-09-21T10:06:17.221 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_einval 2024-09-21T10:06:17.223 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_einval (2 ms) 2024-09-21T10:06:17.223 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_u64_eio 2024-09-21T10:06:17.225 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_u64_eio (2 ms) 2024-09-21T10:06:17.225 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_at_max_keys 2024-09-21T10:06:17.226 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_at_max_keys (1 ms) 2024-09-21T10:06:17.226 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_set_vals_over_max_keys 2024-09-21T10:06:17.227 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_set_vals_over_max_keys (1 ms) 2024-09-21T10:06:17.227 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_str 2024-09-21T10:06:17.233 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_str (6 ms) 2024-09-21T10:06:17.233 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_noexist_u64 2024-09-21T10:06:17.239 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_noexist_u64 (6 ms) 2024-09-21T10:06:17.239 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_str 2024-09-21T10:06:17.246 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_str (7 ms) 2024-09-21T10:06:17.246 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64 2024-09-21T10:06:17.254 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64 (8 ms) 2024-09-21T10:06:17.254 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_einval 2024-09-21T10:06:17.256 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_einval (2 ms) 2024-09-21T10:06:17.256 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_eio 2024-09-21T10:06:17.258 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_eio (2 ms) 2024-09-21T10:06:17.258 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_at_max_keys 2024-09-21T10:06:17.259 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_at_max_keys (1 ms) 2024-09-21T10:06:17.259 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_over_max_keys 2024-09-21T10:06:17.260 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_over_max_keys (1 ms) 2024-09-21T10:06:17.260 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] CmpOmap.cmp_rm_keys_u64_empty 2024-09-21T10:06:17.268 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] CmpOmap.cmp_rm_keys_u64_empty (8 ms) 2024-09-21T10:06:17.268 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 31 tests from CmpOmap (145 ms total) 2024-09-21T10:06:17.268 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:06:17.268 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:06:17.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:17 smithi033 ceph-mon[98546]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T10:06:17.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:17 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2728832761' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:17.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:17 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:17.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:17 smithi033 ceph-mon[98546]: pgmap v996: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:17.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:17 smithi033 ceph-mon[96568]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T10:06:17.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:17 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2728832761' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:17.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:17 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:17.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:17 smithi033 ceph-mon[96568]: pgmap v996: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:17.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:17 smithi105 ceph-mon[86649]: osdmap e892: 8 total, 8 up, 8 in 2024-09-21T10:06:17.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:17 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2728832761' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:17.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:17 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:17.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:17 smithi105 ceph-mon[86649]: pgmap v996: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:18.128 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 31 tests from 1 test suite ran. (2700 ms total) 2024-09-21T10:06:18.128 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 31 tests. 2024-09-21T10:06:18.131 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T10:06:18.131 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:06:18.201 INFO:tasks.workunit:Running workunit cls/test_cls_hello.sh... 2024-09-21T10:06:18.202 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_hello.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_hello.sh 2024-09-21T10:06:18.273 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:06:18.273 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 6 tests from 1 test suite. 2024-09-21T10:06:18.273 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:06:18.273 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 6 tests from ClsHello 2024-09-21T10:06:18.273 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsHello.SayHello 2024-09-21T10:06:18.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:18 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:18.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:18 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:18.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:18 smithi033 ceph-mon[98546]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T10:06:18.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:18 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:18.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:18 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:18.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:18 smithi033 ceph-mon[96568]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T10:06:18.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:18 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:18.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:18 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129224-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:18.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:18 smithi105 ceph-mon[86649]: osdmap e893: 8 total, 8 up, 8 in 2024-09-21T10:06:19.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:19 smithi033 ceph-mon[98546]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T10:06:19.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:19 smithi033 ceph-mon[98546]: pgmap v999: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:19.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:19 smithi033 ceph-mon[96568]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T10:06:19.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:19 smithi033 ceph-mon[96568]: pgmap v999: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:19.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:19 smithi105 ceph-mon[86649]: osdmap e894: 8 total, 8 up, 8 in 2024-09-21T10:06:19.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:19 smithi105 ceph-mon[86649]: pgmap v999: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[98546]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T10:06:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2648140188' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:20.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:20.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:20.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[96568]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T10:06:20.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2648140188' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:20.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:20 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:20.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:20 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:20.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:20 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:20.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:20 smithi105 ceph-mon[86649]: osdmap e895: 8 total, 8 up, 8 in 2024-09-21T10:06:20.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:20 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2648140188' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:20.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:20 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:20.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:06:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:06:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:21.152 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsHello.SayHello (2879 ms) 2024-09-21T10:06:21.152 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsHello.RecordHello 2024-09-21T10:06:21.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:21 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:21.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:21 smithi033 ceph-mon[98546]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T10:06:21.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:21 smithi033 ceph-mon[98546]: pgmap v1002: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:21.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:21 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:21.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:21 smithi033 ceph-mon[96568]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T10:06:21.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:21 smithi033 ceph-mon[96568]: pgmap v1002: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:21.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:21 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:21.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:21 smithi105 ceph-mon[86649]: osdmap e896: 8 total, 8 up, 8 in 2024-09-21T10:06:21.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:21 smithi105 ceph-mon[86649]: pgmap v1002: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:21.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:06:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:06:21] "GET /metrics HTTP/1.1" 200 34091 "" "Prometheus/2.51.0" 2024-09-21T10:06:22.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:22 smithi033 ceph-mon[98546]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T10:06:22.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:22 smithi033 ceph-mon[96568]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T10:06:22.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:22 smithi105 ceph-mon[86649]: osdmap e897: 8 total, 8 up, 8 in 2024-09-21T10:06:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:23 smithi033 ceph-mon[98546]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T10:06:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:23 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/878917466' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:23 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:23.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:23 smithi033 ceph-mon[98546]: pgmap v1005: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:23.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:23 smithi033 ceph-mon[96568]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T10:06:23.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:23 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/878917466' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:23.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:23 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:23.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:23 smithi033 ceph-mon[96568]: pgmap v1005: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:23 smithi105 ceph-mon[86649]: osdmap e898: 8 total, 8 up, 8 in 2024-09-21T10:06:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:23 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/878917466' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:23 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:23.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:23 smithi105 ceph-mon[86649]: pgmap v1005: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:24.164 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsHello.RecordHello (3012 ms) 2024-09-21T10:06:24.165 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsHello.WriteReturnData 2024-09-21T10:06:24.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:24 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:24.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:24 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:24.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:24 smithi033 ceph-mon[98546]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T10:06:24.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:24 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:24.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:24 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:24.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:24 smithi033 ceph-mon[96568]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T10:06:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:24 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:24 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:24.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:24 smithi105 ceph-mon[86649]: osdmap e899: 8 total, 8 up, 8 in 2024-09-21T10:06:25.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:25 smithi033 ceph-mon[98546]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T10:06:25.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:25 smithi033 ceph-mon[98546]: pgmap v1008: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:25.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:25 smithi033 ceph-mon[96568]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T10:06:25.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:25 smithi033 ceph-mon[96568]: pgmap v1008: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:25.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:25 smithi105 ceph-mon[86649]: osdmap e900: 8 total, 8 up, 8 in 2024-09-21T10:06:25.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:25 smithi105 ceph-mon[86649]: pgmap v1008: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:26.202 INFO:tasks.workunit.client.0.smithi033.stdout:require_osd_release = squid 2024-09-21T10:06:26.207 INFO:tasks.workunit.client.0.smithi033.stdout:0 2024-09-21T10:06:26.208 INFO:tasks.workunit.client.0.smithi033.stdout:1 2024-09-21T10:06:26.210 INFO:tasks.workunit.client.0.smithi033.stdout:2 2024-09-21T10:06:26.211 INFO:tasks.workunit.client.0.smithi033.stdout:3 2024-09-21T10:06:26.212 INFO:tasks.workunit.client.0.smithi033.stdout:4 2024-09-21T10:06:26.213 INFO:tasks.workunit.client.0.smithi033.stdout:5 2024-09-21T10:06:26.214 INFO:tasks.workunit.client.0.smithi033.stdout:6 2024-09-21T10:06:26.215 INFO:tasks.workunit.client.0.smithi033.stdout:7 2024-09-21T10:06:26.216 INFO:tasks.workunit.client.0.smithi033.stdout:8 2024-09-21T10:06:26.218 INFO:tasks.workunit.client.0.smithi033.stdout:9 2024-09-21T10:06:26.219 INFO:tasks.workunit.client.0.smithi033.stdout:10 2024-09-21T10:06:26.220 INFO:tasks.workunit.client.0.smithi033.stdout:11 2024-09-21T10:06:26.221 INFO:tasks.workunit.client.0.smithi033.stdout:12 2024-09-21T10:06:26.222 INFO:tasks.workunit.client.0.smithi033.stdout:13 2024-09-21T10:06:26.224 INFO:tasks.workunit.client.0.smithi033.stdout:14 2024-09-21T10:06:26.225 INFO:tasks.workunit.client.0.smithi033.stdout:15 2024-09-21T10:06:26.226 INFO:tasks.workunit.client.0.smithi033.stdout:16 2024-09-21T10:06:26.226 INFO:tasks.workunit.client.0.smithi033.stdout:17 2024-09-21T10:06:26.241 INFO:tasks.workunit.client.0.smithi033.stdout:18 2024-09-21T10:06:26.251 INFO:tasks.workunit.client.0.smithi033.stdout:19 2024-09-21T10:06:26.252 INFO:tasks.workunit.client.0.smithi033.stdout:20 2024-09-21T10:06:26.253 INFO:tasks.workunit.client.0.smithi033.stdout:21 2024-09-21T10:06:26.254 INFO:tasks.workunit.client.0.smithi033.stdout:22 2024-09-21T10:06:26.255 INFO:tasks.workunit.client.0.smithi033.stdout:23 2024-09-21T10:06:26.256 INFO:tasks.workunit.client.0.smithi033.stdout:24 2024-09-21T10:06:26.257 INFO:tasks.workunit.client.0.smithi033.stdout:25 2024-09-21T10:06:26.258 INFO:tasks.workunit.client.0.smithi033.stdout:26 2024-09-21T10:06:26.259 INFO:tasks.workunit.client.0.smithi033.stdout:27 2024-09-21T10:06:26.260 INFO:tasks.workunit.client.0.smithi033.stdout:28 2024-09-21T10:06:26.261 INFO:tasks.workunit.client.0.smithi033.stdout:29 2024-09-21T10:06:26.261 INFO:tasks.workunit.client.0.smithi033.stdout:30 2024-09-21T10:06:26.262 INFO:tasks.workunit.client.0.smithi033.stdout:31 2024-09-21T10:06:26.263 INFO:tasks.workunit.client.0.smithi033.stdout:32 2024-09-21T10:06:26.264 INFO:tasks.workunit.client.0.smithi033.stdout:33 2024-09-21T10:06:26.265 INFO:tasks.workunit.client.0.smithi033.stdout:34 2024-09-21T10:06:26.266 INFO:tasks.workunit.client.0.smithi033.stdout:35 2024-09-21T10:06:26.267 INFO:tasks.workunit.client.0.smithi033.stdout:36 2024-09-21T10:06:26.268 INFO:tasks.workunit.client.0.smithi033.stdout:37 2024-09-21T10:06:26.268 INFO:tasks.workunit.client.0.smithi033.stdout:38 2024-09-21T10:06:26.269 INFO:tasks.workunit.client.0.smithi033.stdout:39 2024-09-21T10:06:26.270 INFO:tasks.workunit.client.0.smithi033.stdout:40 2024-09-21T10:06:26.271 INFO:tasks.workunit.client.0.smithi033.stdout:41 2024-09-21T10:06:26.272 INFO:tasks.workunit.client.0.smithi033.stdout:42 2024-09-21T10:06:26.273 INFO:tasks.workunit.client.0.smithi033.stdout:43 2024-09-21T10:06:26.274 INFO:tasks.workunit.client.0.smithi033.stdout:44 2024-09-21T10:06:26.275 INFO:tasks.workunit.client.0.smithi033.stdout:45 2024-09-21T10:06:26.275 INFO:tasks.workunit.client.0.smithi033.stdout:46 2024-09-21T10:06:26.276 INFO:tasks.workunit.client.0.smithi033.stdout:47 2024-09-21T10:06:26.277 INFO:tasks.workunit.client.0.smithi033.stdout:48 2024-09-21T10:06:26.278 INFO:tasks.workunit.client.0.smithi033.stdout:49 2024-09-21T10:06:26.279 INFO:tasks.workunit.client.0.smithi033.stdout:50 2024-09-21T10:06:26.280 INFO:tasks.workunit.client.0.smithi033.stdout:51 2024-09-21T10:06:26.281 INFO:tasks.workunit.client.0.smithi033.stdout:52 2024-09-21T10:06:26.282 INFO:tasks.workunit.client.0.smithi033.stdout:53 2024-09-21T10:06:26.283 INFO:tasks.workunit.client.0.smithi033.stdout:54 2024-09-21T10:06:26.284 INFO:tasks.workunit.client.0.smithi033.stdout:55 2024-09-21T10:06:26.285 INFO:tasks.workunit.client.0.smithi033.stdout:56 2024-09-21T10:06:26.286 INFO:tasks.workunit.client.0.smithi033.stdout:57 2024-09-21T10:06:26.287 INFO:tasks.workunit.client.0.smithi033.stdout:58 2024-09-21T10:06:26.287 INFO:tasks.workunit.client.0.smithi033.stdout:59 2024-09-21T10:06:26.288 INFO:tasks.workunit.client.0.smithi033.stdout:60 2024-09-21T10:06:26.289 INFO:tasks.workunit.client.0.smithi033.stdout:61 2024-09-21T10:06:26.290 INFO:tasks.workunit.client.0.smithi033.stdout:62 2024-09-21T10:06:26.291 INFO:tasks.workunit.client.0.smithi033.stdout:63 2024-09-21T10:06:26.292 INFO:tasks.workunit.client.0.smithi033.stdout:64 2024-09-21T10:06:26.293 INFO:tasks.workunit.client.0.smithi033.stdout:65 2024-09-21T10:06:26.294 INFO:tasks.workunit.client.0.smithi033.stdout:66 2024-09-21T10:06:26.294 INFO:tasks.workunit.client.0.smithi033.stdout:67 2024-09-21T10:06:26.295 INFO:tasks.workunit.client.0.smithi033.stdout:68 2024-09-21T10:06:26.296 INFO:tasks.workunit.client.0.smithi033.stdout:69 2024-09-21T10:06:26.297 INFO:tasks.workunit.client.0.smithi033.stdout:70 2024-09-21T10:06:26.298 INFO:tasks.workunit.client.0.smithi033.stdout:71 2024-09-21T10:06:26.299 INFO:tasks.workunit.client.0.smithi033.stdout:72 2024-09-21T10:06:26.300 INFO:tasks.workunit.client.0.smithi033.stdout:73 2024-09-21T10:06:26.300 INFO:tasks.workunit.client.0.smithi033.stdout:74 2024-09-21T10:06:26.301 INFO:tasks.workunit.client.0.smithi033.stdout:75 2024-09-21T10:06:26.302 INFO:tasks.workunit.client.0.smithi033.stdout:76 2024-09-21T10:06:26.303 INFO:tasks.workunit.client.0.smithi033.stdout:77 2024-09-21T10:06:26.304 INFO:tasks.workunit.client.0.smithi033.stdout:78 2024-09-21T10:06:26.305 INFO:tasks.workunit.client.0.smithi033.stdout:79 2024-09-21T10:06:26.306 INFO:tasks.workunit.client.0.smithi033.stdout:80 2024-09-21T10:06:26.307 INFO:tasks.workunit.client.0.smithi033.stdout:81 2024-09-21T10:06:26.308 INFO:tasks.workunit.client.0.smithi033.stdout:82 2024-09-21T10:06:26.308 INFO:tasks.workunit.client.0.smithi033.stdout:83 2024-09-21T10:06:26.309 INFO:tasks.workunit.client.0.smithi033.stdout:84 2024-09-21T10:06:26.310 INFO:tasks.workunit.client.0.smithi033.stdout:85 2024-09-21T10:06:26.311 INFO:tasks.workunit.client.0.smithi033.stdout:86 2024-09-21T10:06:26.312 INFO:tasks.workunit.client.0.smithi033.stdout:87 2024-09-21T10:06:26.313 INFO:tasks.workunit.client.0.smithi033.stdout:88 2024-09-21T10:06:26.313 INFO:tasks.workunit.client.0.smithi033.stdout:89 2024-09-21T10:06:26.314 INFO:tasks.workunit.client.0.smithi033.stdout:90 2024-09-21T10:06:26.315 INFO:tasks.workunit.client.0.smithi033.stdout:91 2024-09-21T10:06:26.316 INFO:tasks.workunit.client.0.smithi033.stdout:92 2024-09-21T10:06:26.317 INFO:tasks.workunit.client.0.smithi033.stdout:93 2024-09-21T10:06:26.318 INFO:tasks.workunit.client.0.smithi033.stdout:94 2024-09-21T10:06:26.318 INFO:tasks.workunit.client.0.smithi033.stdout:95 2024-09-21T10:06:26.319 INFO:tasks.workunit.client.0.smithi033.stdout:96 2024-09-21T10:06:26.320 INFO:tasks.workunit.client.0.smithi033.stdout:97 2024-09-21T10:06:26.321 INFO:tasks.workunit.client.0.smithi033.stdout:98 2024-09-21T10:06:26.322 INFO:tasks.workunit.client.0.smithi033.stdout:99 2024-09-21T10:06:26.323 INFO:tasks.workunit.client.0.smithi033.stdout:100 2024-09-21T10:06:26.323 INFO:tasks.workunit.client.0.smithi033.stdout:101 2024-09-21T10:06:26.324 INFO:tasks.workunit.client.0.smithi033.stdout:102 2024-09-21T10:06:26.325 INFO:tasks.workunit.client.0.smithi033.stdout:103 2024-09-21T10:06:26.326 INFO:tasks.workunit.client.0.smithi033.stdout:104 2024-09-21T10:06:26.327 INFO:tasks.workunit.client.0.smithi033.stdout:105 2024-09-21T10:06:26.328 INFO:tasks.workunit.client.0.smithi033.stdout:106 2024-09-21T10:06:26.328 INFO:tasks.workunit.client.0.smithi033.stdout:107 2024-09-21T10:06:26.329 INFO:tasks.workunit.client.0.smithi033.stdout:108 2024-09-21T10:06:26.330 INFO:tasks.workunit.client.0.smithi033.stdout:109 2024-09-21T10:06:26.331 INFO:tasks.workunit.client.0.smithi033.stdout:110 2024-09-21T10:06:26.332 INFO:tasks.workunit.client.0.smithi033.stdout:111 2024-09-21T10:06:26.333 INFO:tasks.workunit.client.0.smithi033.stdout:112 2024-09-21T10:06:26.333 INFO:tasks.workunit.client.0.smithi033.stdout:113 2024-09-21T10:06:26.334 INFO:tasks.workunit.client.0.smithi033.stdout:114 2024-09-21T10:06:26.335 INFO:tasks.workunit.client.0.smithi033.stdout:115 2024-09-21T10:06:26.336 INFO:tasks.workunit.client.0.smithi033.stdout:116 2024-09-21T10:06:26.337 INFO:tasks.workunit.client.0.smithi033.stdout:117 2024-09-21T10:06:26.337 INFO:tasks.workunit.client.0.smithi033.stdout:118 2024-09-21T10:06:26.338 INFO:tasks.workunit.client.0.smithi033.stdout:119 2024-09-21T10:06:26.339 INFO:tasks.workunit.client.0.smithi033.stdout:120 2024-09-21T10:06:26.340 INFO:tasks.workunit.client.0.smithi033.stdout:121 2024-09-21T10:06:26.341 INFO:tasks.workunit.client.0.smithi033.stdout:122 2024-09-21T10:06:26.341 INFO:tasks.workunit.client.0.smithi033.stdout:123 2024-09-21T10:06:26.342 INFO:tasks.workunit.client.0.smithi033.stdout:124 2024-09-21T10:06:26.343 INFO:tasks.workunit.client.0.smithi033.stdout:125 2024-09-21T10:06:26.344 INFO:tasks.workunit.client.0.smithi033.stdout:126 2024-09-21T10:06:26.345 INFO:tasks.workunit.client.0.smithi033.stdout:127 2024-09-21T10:06:26.346 INFO:tasks.workunit.client.0.smithi033.stdout:128 2024-09-21T10:06:26.347 INFO:tasks.workunit.client.0.smithi033.stdout:129 2024-09-21T10:06:26.348 INFO:tasks.workunit.client.0.smithi033.stdout:130 2024-09-21T10:06:26.349 INFO:tasks.workunit.client.0.smithi033.stdout:131 2024-09-21T10:06:26.349 INFO:tasks.workunit.client.0.smithi033.stdout:132 2024-09-21T10:06:26.350 INFO:tasks.workunit.client.0.smithi033.stdout:133 2024-09-21T10:06:26.351 INFO:tasks.workunit.client.0.smithi033.stdout:134 2024-09-21T10:06:26.352 INFO:tasks.workunit.client.0.smithi033.stdout:135 2024-09-21T10:06:26.353 INFO:tasks.workunit.client.0.smithi033.stdout:136 2024-09-21T10:06:26.353 INFO:tasks.workunit.client.0.smithi033.stdout:137 2024-09-21T10:06:26.354 INFO:tasks.workunit.client.0.smithi033.stdout:138 2024-09-21T10:06:26.355 INFO:tasks.workunit.client.0.smithi033.stdout:139 2024-09-21T10:06:26.356 INFO:tasks.workunit.client.0.smithi033.stdout:140 2024-09-21T10:06:26.357 INFO:tasks.workunit.client.0.smithi033.stdout:141 2024-09-21T10:06:26.357 INFO:tasks.workunit.client.0.smithi033.stdout:142 2024-09-21T10:06:26.358 INFO:tasks.workunit.client.0.smithi033.stdout:143 2024-09-21T10:06:26.359 INFO:tasks.workunit.client.0.smithi033.stdout:144 2024-09-21T10:06:26.360 INFO:tasks.workunit.client.0.smithi033.stdout:145 2024-09-21T10:06:26.361 INFO:tasks.workunit.client.0.smithi033.stdout:146 2024-09-21T10:06:26.362 INFO:tasks.workunit.client.0.smithi033.stdout:147 2024-09-21T10:06:26.362 INFO:tasks.workunit.client.0.smithi033.stdout:148 2024-09-21T10:06:26.363 INFO:tasks.workunit.client.0.smithi033.stdout:149 2024-09-21T10:06:26.364 INFO:tasks.workunit.client.0.smithi033.stdout:150 2024-09-21T10:06:26.365 INFO:tasks.workunit.client.0.smithi033.stdout:151 2024-09-21T10:06:26.366 INFO:tasks.workunit.client.0.smithi033.stdout:152 2024-09-21T10:06:26.367 INFO:tasks.workunit.client.0.smithi033.stdout:153 2024-09-21T10:06:26.368 INFO:tasks.workunit.client.0.smithi033.stdout:154 2024-09-21T10:06:26.369 INFO:tasks.workunit.client.0.smithi033.stdout:155 2024-09-21T10:06:26.369 INFO:tasks.workunit.client.0.smithi033.stdout:156 2024-09-21T10:06:26.370 INFO:tasks.workunit.client.0.smithi033.stdout:157 2024-09-21T10:06:26.371 INFO:tasks.workunit.client.0.smithi033.stdout:158 2024-09-21T10:06:26.372 INFO:tasks.workunit.client.0.smithi033.stdout:159 2024-09-21T10:06:26.373 INFO:tasks.workunit.client.0.smithi033.stdout:160 2024-09-21T10:06:26.374 INFO:tasks.workunit.client.0.smithi033.stdout:161 2024-09-21T10:06:26.375 INFO:tasks.workunit.client.0.smithi033.stdout:162 2024-09-21T10:06:26.376 INFO:tasks.workunit.client.0.smithi033.stdout:163 2024-09-21T10:06:26.377 INFO:tasks.workunit.client.0.smithi033.stdout:164 2024-09-21T10:06:26.377 INFO:tasks.workunit.client.0.smithi033.stdout:165 2024-09-21T10:06:26.378 INFO:tasks.workunit.client.0.smithi033.stdout:166 2024-09-21T10:06:26.379 INFO:tasks.workunit.client.0.smithi033.stdout:167 2024-09-21T10:06:26.380 INFO:tasks.workunit.client.0.smithi033.stdout:168 2024-09-21T10:06:26.381 INFO:tasks.workunit.client.0.smithi033.stdout:169 2024-09-21T10:06:26.382 INFO:tasks.workunit.client.0.smithi033.stdout:170 2024-09-21T10:06:26.383 INFO:tasks.workunit.client.0.smithi033.stdout:171 2024-09-21T10:06:26.384 INFO:tasks.workunit.client.0.smithi033.stdout:172 2024-09-21T10:06:26.385 INFO:tasks.workunit.client.0.smithi033.stdout:173 2024-09-21T10:06:26.385 INFO:tasks.workunit.client.0.smithi033.stdout:174 2024-09-21T10:06:26.386 INFO:tasks.workunit.client.0.smithi033.stdout:175 2024-09-21T10:06:26.387 INFO:tasks.workunit.client.0.smithi033.stdout:176 2024-09-21T10:06:26.388 INFO:tasks.workunit.client.0.smithi033.stdout:177 2024-09-21T10:06:26.389 INFO:tasks.workunit.client.0.smithi033.stdout:178 2024-09-21T10:06:26.390 INFO:tasks.workunit.client.0.smithi033.stdout:179 2024-09-21T10:06:26.391 INFO:tasks.workunit.client.0.smithi033.stdout:180 2024-09-21T10:06:26.392 INFO:tasks.workunit.client.0.smithi033.stdout:181 2024-09-21T10:06:26.393 INFO:tasks.workunit.client.0.smithi033.stdout:182 2024-09-21T10:06:26.393 INFO:tasks.workunit.client.0.smithi033.stdout:183 2024-09-21T10:06:26.394 INFO:tasks.workunit.client.0.smithi033.stdout:184 2024-09-21T10:06:26.395 INFO:tasks.workunit.client.0.smithi033.stdout:185 2024-09-21T10:06:26.396 INFO:tasks.workunit.client.0.smithi033.stdout:186 2024-09-21T10:06:26.397 INFO:tasks.workunit.client.0.smithi033.stdout:187 2024-09-21T10:06:26.398 INFO:tasks.workunit.client.0.smithi033.stdout:188 2024-09-21T10:06:26.399 INFO:tasks.workunit.client.0.smithi033.stdout:189 2024-09-21T10:06:26.400 INFO:tasks.workunit.client.0.smithi033.stdout:190 2024-09-21T10:06:26.400 INFO:tasks.workunit.client.0.smithi033.stdout:191 2024-09-21T10:06:26.401 INFO:tasks.workunit.client.0.smithi033.stdout:192 2024-09-21T10:06:26.402 INFO:tasks.workunit.client.0.smithi033.stdout:193 2024-09-21T10:06:26.403 INFO:tasks.workunit.client.0.smithi033.stdout:194 2024-09-21T10:06:26.404 INFO:tasks.workunit.client.0.smithi033.stdout:195 2024-09-21T10:06:26.405 INFO:tasks.workunit.client.0.smithi033.stdout:196 2024-09-21T10:06:26.406 INFO:tasks.workunit.client.0.smithi033.stdout:197 2024-09-21T10:06:26.406 INFO:tasks.workunit.client.0.smithi033.stdout:198 2024-09-21T10:06:26.407 INFO:tasks.workunit.client.0.smithi033.stdout:199 2024-09-21T10:06:26.408 INFO:tasks.workunit.client.0.smithi033.stdout:200 2024-09-21T10:06:26.409 INFO:tasks.workunit.client.0.smithi033.stdout:201 2024-09-21T10:06:26.410 INFO:tasks.workunit.client.0.smithi033.stdout:202 2024-09-21T10:06:26.411 INFO:tasks.workunit.client.0.smithi033.stdout:203 2024-09-21T10:06:26.412 INFO:tasks.workunit.client.0.smithi033.stdout:204 2024-09-21T10:06:26.412 INFO:tasks.workunit.client.0.smithi033.stdout:205 2024-09-21T10:06:26.413 INFO:tasks.workunit.client.0.smithi033.stdout:206 2024-09-21T10:06:26.414 INFO:tasks.workunit.client.0.smithi033.stdout:207 2024-09-21T10:06:26.415 INFO:tasks.workunit.client.0.smithi033.stdout:208 2024-09-21T10:06:26.416 INFO:tasks.workunit.client.0.smithi033.stdout:209 2024-09-21T10:06:26.417 INFO:tasks.workunit.client.0.smithi033.stdout:210 2024-09-21T10:06:26.418 INFO:tasks.workunit.client.0.smithi033.stdout:211 2024-09-21T10:06:26.419 INFO:tasks.workunit.client.0.smithi033.stdout:212 2024-09-21T10:06:26.420 INFO:tasks.workunit.client.0.smithi033.stdout:213 2024-09-21T10:06:26.421 INFO:tasks.workunit.client.0.smithi033.stdout:214 2024-09-21T10:06:26.421 INFO:tasks.workunit.client.0.smithi033.stdout:215 2024-09-21T10:06:26.422 INFO:tasks.workunit.client.0.smithi033.stdout:216 2024-09-21T10:06:26.423 INFO:tasks.workunit.client.0.smithi033.stdout:217 2024-09-21T10:06:26.424 INFO:tasks.workunit.client.0.smithi033.stdout:218 2024-09-21T10:06:26.425 INFO:tasks.workunit.client.0.smithi033.stdout:219 2024-09-21T10:06:26.426 INFO:tasks.workunit.client.0.smithi033.stdout:220 2024-09-21T10:06:26.427 INFO:tasks.workunit.client.0.smithi033.stdout:221 2024-09-21T10:06:26.428 INFO:tasks.workunit.client.0.smithi033.stdout:222 2024-09-21T10:06:26.428 INFO:tasks.workunit.client.0.smithi033.stdout:223 2024-09-21T10:06:26.429 INFO:tasks.workunit.client.0.smithi033.stdout:224 2024-09-21T10:06:26.430 INFO:tasks.workunit.client.0.smithi033.stdout:225 2024-09-21T10:06:26.431 INFO:tasks.workunit.client.0.smithi033.stdout:226 2024-09-21T10:06:26.432 INFO:tasks.workunit.client.0.smithi033.stdout:227 2024-09-21T10:06:26.433 INFO:tasks.workunit.client.0.smithi033.stdout:228 2024-09-21T10:06:26.433 INFO:tasks.workunit.client.0.smithi033.stdout:229 2024-09-21T10:06:26.434 INFO:tasks.workunit.client.0.smithi033.stdout:230 2024-09-21T10:06:26.435 INFO:tasks.workunit.client.0.smithi033.stdout:231 2024-09-21T10:06:26.436 INFO:tasks.workunit.client.0.smithi033.stdout:232 2024-09-21T10:06:26.437 INFO:tasks.workunit.client.0.smithi033.stdout:233 2024-09-21T10:06:26.438 INFO:tasks.workunit.client.0.smithi033.stdout:234 2024-09-21T10:06:26.439 INFO:tasks.workunit.client.0.smithi033.stdout:235 2024-09-21T10:06:26.439 INFO:tasks.workunit.client.0.smithi033.stdout:236 2024-09-21T10:06:26.440 INFO:tasks.workunit.client.0.smithi033.stdout:237 2024-09-21T10:06:26.441 INFO:tasks.workunit.client.0.smithi033.stdout:238 2024-09-21T10:06:26.442 INFO:tasks.workunit.client.0.smithi033.stdout:239 2024-09-21T10:06:26.443 INFO:tasks.workunit.client.0.smithi033.stdout:240 2024-09-21T10:06:26.444 INFO:tasks.workunit.client.0.smithi033.stdout:241 2024-09-21T10:06:26.445 INFO:tasks.workunit.client.0.smithi033.stdout:242 2024-09-21T10:06:26.445 INFO:tasks.workunit.client.0.smithi033.stdout:243 2024-09-21T10:06:26.446 INFO:tasks.workunit.client.0.smithi033.stdout:244 2024-09-21T10:06:26.447 INFO:tasks.workunit.client.0.smithi033.stdout:245 2024-09-21T10:06:26.448 INFO:tasks.workunit.client.0.smithi033.stdout:246 2024-09-21T10:06:26.449 INFO:tasks.workunit.client.0.smithi033.stdout:247 2024-09-21T10:06:26.450 INFO:tasks.workunit.client.0.smithi033.stdout:248 2024-09-21T10:06:26.450 INFO:tasks.workunit.client.0.smithi033.stdout:249 2024-09-21T10:06:26.451 INFO:tasks.workunit.client.0.smithi033.stdout:250 2024-09-21T10:06:26.452 INFO:tasks.workunit.client.0.smithi033.stdout:251 2024-09-21T10:06:26.453 INFO:tasks.workunit.client.0.smithi033.stdout:252 2024-09-21T10:06:26.454 INFO:tasks.workunit.client.0.smithi033.stdout:253 2024-09-21T10:06:26.455 INFO:tasks.workunit.client.0.smithi033.stdout:254 2024-09-21T10:06:26.456 INFO:tasks.workunit.client.0.smithi033.stdout:255 2024-09-21T10:06:26.457 INFO:tasks.workunit.client.0.smithi033.stdout:256 2024-09-21T10:06:26.457 INFO:tasks.workunit.client.0.smithi033.stdout:257 2024-09-21T10:06:26.458 INFO:tasks.workunit.client.0.smithi033.stdout:258 2024-09-21T10:06:26.459 INFO:tasks.workunit.client.0.smithi033.stdout:259 2024-09-21T10:06:26.460 INFO:tasks.workunit.client.0.smithi033.stdout:260 2024-09-21T10:06:26.461 INFO:tasks.workunit.client.0.smithi033.stdout:261 2024-09-21T10:06:26.462 INFO:tasks.workunit.client.0.smithi033.stdout:262 2024-09-21T10:06:26.463 INFO:tasks.workunit.client.0.smithi033.stdout:263 2024-09-21T10:06:26.464 INFO:tasks.workunit.client.0.smithi033.stdout:264 2024-09-21T10:06:26.464 INFO:tasks.workunit.client.0.smithi033.stdout:265 2024-09-21T10:06:26.465 INFO:tasks.workunit.client.0.smithi033.stdout:266 2024-09-21T10:06:26.466 INFO:tasks.workunit.client.0.smithi033.stdout:267 2024-09-21T10:06:26.467 INFO:tasks.workunit.client.0.smithi033.stdout:268 2024-09-21T10:06:26.468 INFO:tasks.workunit.client.0.smithi033.stdout:269 2024-09-21T10:06:26.469 INFO:tasks.workunit.client.0.smithi033.stdout:270 2024-09-21T10:06:26.470 INFO:tasks.workunit.client.0.smithi033.stdout:271 2024-09-21T10:06:26.471 INFO:tasks.workunit.client.0.smithi033.stdout:272 2024-09-21T10:06:26.472 INFO:tasks.workunit.client.0.smithi033.stdout:273 2024-09-21T10:06:26.473 INFO:tasks.workunit.client.0.smithi033.stdout:274 2024-09-21T10:06:26.474 INFO:tasks.workunit.client.0.smithi033.stdout:275 2024-09-21T10:06:26.474 INFO:tasks.workunit.client.0.smithi033.stdout:276 2024-09-21T10:06:26.475 INFO:tasks.workunit.client.0.smithi033.stdout:277 2024-09-21T10:06:26.476 INFO:tasks.workunit.client.0.smithi033.stdout:278 2024-09-21T10:06:26.477 INFO:tasks.workunit.client.0.smithi033.stdout:279 2024-09-21T10:06:26.478 INFO:tasks.workunit.client.0.smithi033.stdout:280 2024-09-21T10:06:26.479 INFO:tasks.workunit.client.0.smithi033.stdout:281 2024-09-21T10:06:26.480 INFO:tasks.workunit.client.0.smithi033.stdout:282 2024-09-21T10:06:26.481 INFO:tasks.workunit.client.0.smithi033.stdout:283 2024-09-21T10:06:26.482 INFO:tasks.workunit.client.0.smithi033.stdout:284 2024-09-21T10:06:26.483 INFO:tasks.workunit.client.0.smithi033.stdout:285 2024-09-21T10:06:26.483 INFO:tasks.workunit.client.0.smithi033.stdout:286 2024-09-21T10:06:26.484 INFO:tasks.workunit.client.0.smithi033.stdout:287 2024-09-21T10:06:26.485 INFO:tasks.workunit.client.0.smithi033.stdout:288 2024-09-21T10:06:26.486 INFO:tasks.workunit.client.0.smithi033.stdout:289 2024-09-21T10:06:26.487 INFO:tasks.workunit.client.0.smithi033.stdout:290 2024-09-21T10:06:26.488 INFO:tasks.workunit.client.0.smithi033.stdout:291 2024-09-21T10:06:26.489 INFO:tasks.workunit.client.0.smithi033.stdout:292 2024-09-21T10:06:26.490 INFO:tasks.workunit.client.0.smithi033.stdout:293 2024-09-21T10:06:26.491 INFO:tasks.workunit.client.0.smithi033.stdout:294 2024-09-21T10:06:26.492 INFO:tasks.workunit.client.0.smithi033.stdout:295 2024-09-21T10:06:26.493 INFO:tasks.workunit.client.0.smithi033.stdout:296 2024-09-21T10:06:26.494 INFO:tasks.workunit.client.0.smithi033.stdout:297 2024-09-21T10:06:26.495 INFO:tasks.workunit.client.0.smithi033.stdout:298 2024-09-21T10:06:26.495 INFO:tasks.workunit.client.0.smithi033.stdout:299 2024-09-21T10:06:26.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:26.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:26.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[98546]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T10:06:26.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/299527596' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:26.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:26.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:26.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:26.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[96568]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T10:06:26.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/299527596' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:26.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:26 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:26.497 INFO:tasks.workunit.client.0.smithi033.stdout:300 2024-09-21T10:06:26.497 INFO:tasks.workunit.client.0.smithi033.stdout:301 2024-09-21T10:06:26.498 INFO:tasks.workunit.client.0.smithi033.stdout:302 2024-09-21T10:06:26.499 INFO:tasks.workunit.client.0.smithi033.stdout:303 2024-09-21T10:06:26.500 INFO:tasks.workunit.client.0.smithi033.stdout:304 2024-09-21T10:06:26.501 INFO:tasks.workunit.client.0.smithi033.stdout:305 2024-09-21T10:06:26.501 INFO:tasks.workunit.client.0.smithi033.stdout:306 2024-09-21T10:06:26.502 INFO:tasks.workunit.client.0.smithi033.stdout:307 2024-09-21T10:06:26.503 INFO:tasks.workunit.client.0.smithi033.stdout:308 2024-09-21T10:06:26.504 INFO:tasks.workunit.client.0.smithi033.stdout:309 2024-09-21T10:06:26.505 INFO:tasks.workunit.client.0.smithi033.stdout:310 2024-09-21T10:06:26.506 INFO:tasks.workunit.client.0.smithi033.stdout:311 2024-09-21T10:06:26.507 INFO:tasks.workunit.client.0.smithi033.stdout:312 2024-09-21T10:06:26.508 INFO:tasks.workunit.client.0.smithi033.stdout:313 2024-09-21T10:06:26.509 INFO:tasks.workunit.client.0.smithi033.stdout:314 2024-09-21T10:06:26.509 INFO:tasks.workunit.client.0.smithi033.stdout:315 2024-09-21T10:06:26.510 INFO:tasks.workunit.client.0.smithi033.stdout:316 2024-09-21T10:06:26.511 INFO:tasks.workunit.client.0.smithi033.stdout:317 2024-09-21T10:06:26.512 INFO:tasks.workunit.client.0.smithi033.stdout:318 2024-09-21T10:06:26.513 INFO:tasks.workunit.client.0.smithi033.stdout:319 2024-09-21T10:06:26.514 INFO:tasks.workunit.client.0.smithi033.stdout:320 2024-09-21T10:06:26.515 INFO:tasks.workunit.client.0.smithi033.stdout:321 2024-09-21T10:06:26.516 INFO:tasks.workunit.client.0.smithi033.stdout:322 2024-09-21T10:06:26.516 INFO:tasks.workunit.client.0.smithi033.stdout:323 2024-09-21T10:06:26.518 INFO:tasks.workunit.client.0.smithi033.stdout:324 2024-09-21T10:06:26.518 INFO:tasks.workunit.client.0.smithi033.stdout:325 2024-09-21T10:06:26.519 INFO:tasks.workunit.client.0.smithi033.stdout:326 2024-09-21T10:06:26.520 INFO:tasks.workunit.client.0.smithi033.stdout:327 2024-09-21T10:06:26.521 INFO:tasks.workunit.client.0.smithi033.stdout:328 2024-09-21T10:06:26.522 INFO:tasks.workunit.client.0.smithi033.stdout:329 2024-09-21T10:06:26.523 INFO:tasks.workunit.client.0.smithi033.stdout:330 2024-09-21T10:06:26.524 INFO:tasks.workunit.client.0.smithi033.stdout:331 2024-09-21T10:06:26.525 INFO:tasks.workunit.client.0.smithi033.stdout:332 2024-09-21T10:06:26.526 INFO:tasks.workunit.client.0.smithi033.stdout:333 2024-09-21T10:06:26.526 INFO:tasks.workunit.client.0.smithi033.stdout:334 2024-09-21T10:06:26.527 INFO:tasks.workunit.client.0.smithi033.stdout:335 2024-09-21T10:06:26.528 INFO:tasks.workunit.client.0.smithi033.stdout:336 2024-09-21T10:06:26.529 INFO:tasks.workunit.client.0.smithi033.stdout:337 2024-09-21T10:06:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:26 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:26 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:26 smithi105 ceph-mon[86649]: osdmap e901: 8 total, 8 up, 8 in 2024-09-21T10:06:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:26 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/299527596' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:26.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:26 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:26.530 INFO:tasks.workunit.client.0.smithi033.stdout:338 2024-09-21T10:06:26.531 INFO:tasks.workunit.client.0.smithi033.stdout:339 2024-09-21T10:06:26.534 INFO:tasks.workunit.client.0.smithi033.stdout:340 2024-09-21T10:06:26.535 INFO:tasks.workunit.client.0.smithi033.stdout:341 2024-09-21T10:06:26.535 INFO:tasks.workunit.client.0.smithi033.stdout:342 2024-09-21T10:06:26.535 INFO:tasks.workunit.client.0.smithi033.stdout:343 2024-09-21T10:06:26.536 INFO:tasks.workunit.client.0.smithi033.stdout:344 2024-09-21T10:06:26.537 INFO:tasks.workunit.client.0.smithi033.stdout:345 2024-09-21T10:06:26.538 INFO:tasks.workunit.client.0.smithi033.stdout:346 2024-09-21T10:06:26.538 INFO:tasks.workunit.client.0.smithi033.stdout:347 2024-09-21T10:06:26.539 INFO:tasks.workunit.client.0.smithi033.stdout:348 2024-09-21T10:06:26.540 INFO:tasks.workunit.client.0.smithi033.stdout:349 2024-09-21T10:06:26.541 INFO:tasks.workunit.client.0.smithi033.stdout:350 2024-09-21T10:06:26.542 INFO:tasks.workunit.client.0.smithi033.stdout:351 2024-09-21T10:06:26.543 INFO:tasks.workunit.client.0.smithi033.stdout:352 2024-09-21T10:06:26.544 INFO:tasks.workunit.client.0.smithi033.stdout:353 2024-09-21T10:06:26.545 INFO:tasks.workunit.client.0.smithi033.stdout:354 2024-09-21T10:06:26.546 INFO:tasks.workunit.client.0.smithi033.stdout:355 2024-09-21T10:06:26.547 INFO:tasks.workunit.client.0.smithi033.stdout:356 2024-09-21T10:06:26.547 INFO:tasks.workunit.client.0.smithi033.stdout:357 2024-09-21T10:06:26.548 INFO:tasks.workunit.client.0.smithi033.stdout:358 2024-09-21T10:06:26.549 INFO:tasks.workunit.client.0.smithi033.stdout:359 2024-09-21T10:06:26.550 INFO:tasks.workunit.client.0.smithi033.stdout:360 2024-09-21T10:06:26.551 INFO:tasks.workunit.client.0.smithi033.stdout:361 2024-09-21T10:06:26.552 INFO:tasks.workunit.client.0.smithi033.stdout:362 2024-09-21T10:06:26.553 INFO:tasks.workunit.client.0.smithi033.stdout:363 2024-09-21T10:06:26.554 INFO:tasks.workunit.client.0.smithi033.stdout:364 2024-09-21T10:06:26.555 INFO:tasks.workunit.client.0.smithi033.stdout:365 2024-09-21T10:06:26.556 INFO:tasks.workunit.client.0.smithi033.stdout:366 2024-09-21T10:06:26.556 INFO:tasks.workunit.client.0.smithi033.stdout:367 2024-09-21T10:06:26.557 INFO:tasks.workunit.client.0.smithi033.stdout:368 2024-09-21T10:06:26.558 INFO:tasks.workunit.client.0.smithi033.stdout:369 2024-09-21T10:06:26.559 INFO:tasks.workunit.client.0.smithi033.stdout:370 2024-09-21T10:06:26.560 INFO:tasks.workunit.client.0.smithi033.stdout:371 2024-09-21T10:06:26.561 INFO:tasks.workunit.client.0.smithi033.stdout:372 2024-09-21T10:06:26.562 INFO:tasks.workunit.client.0.smithi033.stdout:373 2024-09-21T10:06:26.563 INFO:tasks.workunit.client.0.smithi033.stdout:374 2024-09-21T10:06:26.564 INFO:tasks.workunit.client.0.smithi033.stdout:375 2024-09-21T10:06:26.564 INFO:tasks.workunit.client.0.smithi033.stdout:376 2024-09-21T10:06:26.565 INFO:tasks.workunit.client.0.smithi033.stdout:377 2024-09-21T10:06:26.566 INFO:tasks.workunit.client.0.smithi033.stdout:378 2024-09-21T10:06:26.567 INFO:tasks.workunit.client.0.smithi033.stdout:379 2024-09-21T10:06:26.568 INFO:tasks.workunit.client.0.smithi033.stdout:380 2024-09-21T10:06:26.569 INFO:tasks.workunit.client.0.smithi033.stdout:381 2024-09-21T10:06:26.570 INFO:tasks.workunit.client.0.smithi033.stdout:382 2024-09-21T10:06:26.571 INFO:tasks.workunit.client.0.smithi033.stdout:383 2024-09-21T10:06:26.572 INFO:tasks.workunit.client.0.smithi033.stdout:384 2024-09-21T10:06:26.572 INFO:tasks.workunit.client.0.smithi033.stdout:385 2024-09-21T10:06:26.574 INFO:tasks.workunit.client.0.smithi033.stdout:386 2024-09-21T10:06:26.575 INFO:tasks.workunit.client.0.smithi033.stdout:387 2024-09-21T10:06:26.576 INFO:tasks.workunit.client.0.smithi033.stdout:388 2024-09-21T10:06:26.576 INFO:tasks.workunit.client.0.smithi033.stdout:389 2024-09-21T10:06:26.577 INFO:tasks.workunit.client.0.smithi033.stdout:390 2024-09-21T10:06:26.578 INFO:tasks.workunit.client.0.smithi033.stdout:391 2024-09-21T10:06:26.579 INFO:tasks.workunit.client.0.smithi033.stdout:392 2024-09-21T10:06:26.580 INFO:tasks.workunit.client.0.smithi033.stdout:393 2024-09-21T10:06:26.581 INFO:tasks.workunit.client.0.smithi033.stdout:394 2024-09-21T10:06:26.582 INFO:tasks.workunit.client.0.smithi033.stdout:395 2024-09-21T10:06:26.583 INFO:tasks.workunit.client.0.smithi033.stdout:396 2024-09-21T10:06:26.584 INFO:tasks.workunit.client.0.smithi033.stdout:397 2024-09-21T10:06:26.585 INFO:tasks.workunit.client.0.smithi033.stdout:398 2024-09-21T10:06:26.585 INFO:tasks.workunit.client.0.smithi033.stdout:399 2024-09-21T10:06:26.586 INFO:tasks.workunit.client.0.smithi033.stdout:400 2024-09-21T10:06:26.587 INFO:tasks.workunit.client.0.smithi033.stdout:401 2024-09-21T10:06:26.588 INFO:tasks.workunit.client.0.smithi033.stdout:402 2024-09-21T10:06:26.589 INFO:tasks.workunit.client.0.smithi033.stdout:403 2024-09-21T10:06:26.590 INFO:tasks.workunit.client.0.smithi033.stdout:404 2024-09-21T10:06:26.591 INFO:tasks.workunit.client.0.smithi033.stdout:405 2024-09-21T10:06:26.592 INFO:tasks.workunit.client.0.smithi033.stdout:406 2024-09-21T10:06:26.593 INFO:tasks.workunit.client.0.smithi033.stdout:407 2024-09-21T10:06:26.594 INFO:tasks.workunit.client.0.smithi033.stdout:408 2024-09-21T10:06:26.595 INFO:tasks.workunit.client.0.smithi033.stdout:409 2024-09-21T10:06:26.596 INFO:tasks.workunit.client.0.smithi033.stdout:410 2024-09-21T10:06:26.597 INFO:tasks.workunit.client.0.smithi033.stdout:411 2024-09-21T10:06:26.598 INFO:tasks.workunit.client.0.smithi033.stdout:412 2024-09-21T10:06:26.598 INFO:tasks.workunit.client.0.smithi033.stdout:413 2024-09-21T10:06:26.599 INFO:tasks.workunit.client.0.smithi033.stdout:414 2024-09-21T10:06:26.600 INFO:tasks.workunit.client.0.smithi033.stdout:415 2024-09-21T10:06:26.601 INFO:tasks.workunit.client.0.smithi033.stdout:416 2024-09-21T10:06:26.602 INFO:tasks.workunit.client.0.smithi033.stdout:417 2024-09-21T10:06:26.603 INFO:tasks.workunit.client.0.smithi033.stdout:418 2024-09-21T10:06:26.604 INFO:tasks.workunit.client.0.smithi033.stdout:419 2024-09-21T10:06:26.605 INFO:tasks.workunit.client.0.smithi033.stdout:420 2024-09-21T10:06:26.606 INFO:tasks.workunit.client.0.smithi033.stdout:421 2024-09-21T10:06:26.607 INFO:tasks.workunit.client.0.smithi033.stdout:422 2024-09-21T10:06:26.608 INFO:tasks.workunit.client.0.smithi033.stdout:423 2024-09-21T10:06:26.609 INFO:tasks.workunit.client.0.smithi033.stdout:424 2024-09-21T10:06:26.610 INFO:tasks.workunit.client.0.smithi033.stdout:425 2024-09-21T10:06:26.611 INFO:tasks.workunit.client.0.smithi033.stdout:426 2024-09-21T10:06:26.612 INFO:tasks.workunit.client.0.smithi033.stdout:427 2024-09-21T10:06:26.613 INFO:tasks.workunit.client.0.smithi033.stdout:428 2024-09-21T10:06:26.613 INFO:tasks.workunit.client.0.smithi033.stdout:429 2024-09-21T10:06:26.614 INFO:tasks.workunit.client.0.smithi033.stdout:430 2024-09-21T10:06:26.615 INFO:tasks.workunit.client.0.smithi033.stdout:431 2024-09-21T10:06:26.616 INFO:tasks.workunit.client.0.smithi033.stdout:432 2024-09-21T10:06:26.618 INFO:tasks.workunit.client.0.smithi033.stdout:433 2024-09-21T10:06:26.619 INFO:tasks.workunit.client.0.smithi033.stdout:434 2024-09-21T10:06:26.620 INFO:tasks.workunit.client.0.smithi033.stdout:435 2024-09-21T10:06:26.621 INFO:tasks.workunit.client.0.smithi033.stdout:436 2024-09-21T10:06:26.622 INFO:tasks.workunit.client.0.smithi033.stdout:437 2024-09-21T10:06:26.623 INFO:tasks.workunit.client.0.smithi033.stdout:438 2024-09-21T10:06:26.624 INFO:tasks.workunit.client.0.smithi033.stdout:439 2024-09-21T10:06:26.625 INFO:tasks.workunit.client.0.smithi033.stdout:440 2024-09-21T10:06:26.626 INFO:tasks.workunit.client.0.smithi033.stdout:441 2024-09-21T10:06:26.627 INFO:tasks.workunit.client.0.smithi033.stdout:442 2024-09-21T10:06:26.628 INFO:tasks.workunit.client.0.smithi033.stdout:443 2024-09-21T10:06:26.629 INFO:tasks.workunit.client.0.smithi033.stdout:444 2024-09-21T10:06:26.630 INFO:tasks.workunit.client.0.smithi033.stdout:445 2024-09-21T10:06:26.631 INFO:tasks.workunit.client.0.smithi033.stdout:446 2024-09-21T10:06:26.632 INFO:tasks.workunit.client.0.smithi033.stdout:447 2024-09-21T10:06:26.633 INFO:tasks.workunit.client.0.smithi033.stdout:448 2024-09-21T10:06:26.634 INFO:tasks.workunit.client.0.smithi033.stdout:449 2024-09-21T10:06:26.635 INFO:tasks.workunit.client.0.smithi033.stdout:450 2024-09-21T10:06:26.636 INFO:tasks.workunit.client.0.smithi033.stdout:451 2024-09-21T10:06:26.637 INFO:tasks.workunit.client.0.smithi033.stdout:452 2024-09-21T10:06:26.638 INFO:tasks.workunit.client.0.smithi033.stdout:453 2024-09-21T10:06:26.639 INFO:tasks.workunit.client.0.smithi033.stdout:454 2024-09-21T10:06:26.640 INFO:tasks.workunit.client.0.smithi033.stdout:455 2024-09-21T10:06:26.641 INFO:tasks.workunit.client.0.smithi033.stdout:456 2024-09-21T10:06:26.642 INFO:tasks.workunit.client.0.smithi033.stdout:457 2024-09-21T10:06:26.643 INFO:tasks.workunit.client.0.smithi033.stdout:458 2024-09-21T10:06:26.644 INFO:tasks.workunit.client.0.smithi033.stdout:459 2024-09-21T10:06:26.645 INFO:tasks.workunit.client.0.smithi033.stdout:460 2024-09-21T10:06:26.646 INFO:tasks.workunit.client.0.smithi033.stdout:461 2024-09-21T10:06:26.647 INFO:tasks.workunit.client.0.smithi033.stdout:462 2024-09-21T10:06:26.648 INFO:tasks.workunit.client.0.smithi033.stdout:463 2024-09-21T10:06:26.649 INFO:tasks.workunit.client.0.smithi033.stdout:464 2024-09-21T10:06:26.650 INFO:tasks.workunit.client.0.smithi033.stdout:465 2024-09-21T10:06:26.651 INFO:tasks.workunit.client.0.smithi033.stdout:466 2024-09-21T10:06:26.652 INFO:tasks.workunit.client.0.smithi033.stdout:467 2024-09-21T10:06:26.653 INFO:tasks.workunit.client.0.smithi033.stdout:468 2024-09-21T10:06:26.654 INFO:tasks.workunit.client.0.smithi033.stdout:469 2024-09-21T10:06:26.655 INFO:tasks.workunit.client.0.smithi033.stdout:470 2024-09-21T10:06:26.656 INFO:tasks.workunit.client.0.smithi033.stdout:471 2024-09-21T10:06:26.657 INFO:tasks.workunit.client.0.smithi033.stdout:472 2024-09-21T10:06:26.658 INFO:tasks.workunit.client.0.smithi033.stdout:473 2024-09-21T10:06:26.659 INFO:tasks.workunit.client.0.smithi033.stdout:474 2024-09-21T10:06:26.660 INFO:tasks.workunit.client.0.smithi033.stdout:475 2024-09-21T10:06:26.661 INFO:tasks.workunit.client.0.smithi033.stdout:476 2024-09-21T10:06:26.662 INFO:tasks.workunit.client.0.smithi033.stdout:477 2024-09-21T10:06:26.663 INFO:tasks.workunit.client.0.smithi033.stdout:478 2024-09-21T10:06:26.664 INFO:tasks.workunit.client.0.smithi033.stdout:479 2024-09-21T10:06:26.665 INFO:tasks.workunit.client.0.smithi033.stdout:480 2024-09-21T10:06:26.666 INFO:tasks.workunit.client.0.smithi033.stdout:481 2024-09-21T10:06:26.667 INFO:tasks.workunit.client.0.smithi033.stdout:482 2024-09-21T10:06:26.668 INFO:tasks.workunit.client.0.smithi033.stdout:483 2024-09-21T10:06:26.668 INFO:tasks.workunit.client.0.smithi033.stdout:484 2024-09-21T10:06:26.670 INFO:tasks.workunit.client.0.smithi033.stdout:485 2024-09-21T10:06:26.670 INFO:tasks.workunit.client.0.smithi033.stdout:486 2024-09-21T10:06:26.671 INFO:tasks.workunit.client.0.smithi033.stdout:487 2024-09-21T10:06:26.672 INFO:tasks.workunit.client.0.smithi033.stdout:488 2024-09-21T10:06:26.673 INFO:tasks.workunit.client.0.smithi033.stdout:489 2024-09-21T10:06:26.674 INFO:tasks.workunit.client.0.smithi033.stdout:490 2024-09-21T10:06:26.675 INFO:tasks.workunit.client.0.smithi033.stdout:491 2024-09-21T10:06:26.677 INFO:tasks.workunit.client.0.smithi033.stdout:492 2024-09-21T10:06:26.678 INFO:tasks.workunit.client.0.smithi033.stdout:493 2024-09-21T10:06:26.679 INFO:tasks.workunit.client.0.smithi033.stdout:494 2024-09-21T10:06:26.680 INFO:tasks.workunit.client.0.smithi033.stdout:495 2024-09-21T10:06:26.681 INFO:tasks.workunit.client.0.smithi033.stdout:496 2024-09-21T10:06:26.682 INFO:tasks.workunit.client.0.smithi033.stdout:497 2024-09-21T10:06:26.682 INFO:tasks.workunit.client.0.smithi033.stdout:498 2024-09-21T10:06:26.683 INFO:tasks.workunit.client.0.smithi033.stdout:499 2024-09-21T10:06:26.684 INFO:tasks.workunit.client.0.smithi033.stdout:500 2024-09-21T10:06:26.685 INFO:tasks.workunit.client.0.smithi033.stdout:501 2024-09-21T10:06:26.686 INFO:tasks.workunit.client.0.smithi033.stdout:502 2024-09-21T10:06:26.687 INFO:tasks.workunit.client.0.smithi033.stdout:503 2024-09-21T10:06:26.688 INFO:tasks.workunit.client.0.smithi033.stdout:504 2024-09-21T10:06:26.689 INFO:tasks.workunit.client.0.smithi033.stdout:505 2024-09-21T10:06:26.690 INFO:tasks.workunit.client.0.smithi033.stdout:506 2024-09-21T10:06:26.691 INFO:tasks.workunit.client.0.smithi033.stdout:507 2024-09-21T10:06:26.692 INFO:tasks.workunit.client.0.smithi033.stdout:508 2024-09-21T10:06:26.693 INFO:tasks.workunit.client.0.smithi033.stdout:509 2024-09-21T10:06:26.694 INFO:tasks.workunit.client.0.smithi033.stdout:510 2024-09-21T10:06:26.695 INFO:tasks.workunit.client.0.smithi033.stdout:511 2024-09-21T10:06:26.696 INFO:tasks.workunit.client.0.smithi033.stdout:512 2024-09-21T10:06:26.697 INFO:tasks.workunit.client.0.smithi033.stdout:513 2024-09-21T10:06:26.698 INFO:tasks.workunit.client.0.smithi033.stdout:514 2024-09-21T10:06:26.699 INFO:tasks.workunit.client.0.smithi033.stdout:515 2024-09-21T10:06:26.700 INFO:tasks.workunit.client.0.smithi033.stdout:516 2024-09-21T10:06:26.701 INFO:tasks.workunit.client.0.smithi033.stdout:517 2024-09-21T10:06:26.702 INFO:tasks.workunit.client.0.smithi033.stdout:518 2024-09-21T10:06:26.703 INFO:tasks.workunit.client.0.smithi033.stdout:519 2024-09-21T10:06:26.703 INFO:tasks.workunit.client.0.smithi033.stdout:520 2024-09-21T10:06:26.705 INFO:tasks.workunit.client.0.smithi033.stdout:521 2024-09-21T10:06:26.706 INFO:tasks.workunit.client.0.smithi033.stdout:522 2024-09-21T10:06:26.706 INFO:tasks.workunit.client.0.smithi033.stdout:523 2024-09-21T10:06:26.707 INFO:tasks.workunit.client.0.smithi033.stdout:524 2024-09-21T10:06:26.708 INFO:tasks.workunit.client.0.smithi033.stdout:525 2024-09-21T10:06:26.709 INFO:tasks.workunit.client.0.smithi033.stdout:526 2024-09-21T10:06:26.710 INFO:tasks.workunit.client.0.smithi033.stdout:527 2024-09-21T10:06:26.711 INFO:tasks.workunit.client.0.smithi033.stdout:528 2024-09-21T10:06:26.712 INFO:tasks.workunit.client.0.smithi033.stdout:529 2024-09-21T10:06:26.713 INFO:tasks.workunit.client.0.smithi033.stdout:530 2024-09-21T10:06:26.714 INFO:tasks.workunit.client.0.smithi033.stdout:531 2024-09-21T10:06:26.715 INFO:tasks.workunit.client.0.smithi033.stdout:532 2024-09-21T10:06:26.716 INFO:tasks.workunit.client.0.smithi033.stdout:533 2024-09-21T10:06:26.717 INFO:tasks.workunit.client.0.smithi033.stdout:534 2024-09-21T10:06:26.718 INFO:tasks.workunit.client.0.smithi033.stdout:535 2024-09-21T10:06:26.719 INFO:tasks.workunit.client.0.smithi033.stdout:536 2024-09-21T10:06:26.720 INFO:tasks.workunit.client.0.smithi033.stdout:537 2024-09-21T10:06:26.721 INFO:tasks.workunit.client.0.smithi033.stdout:538 2024-09-21T10:06:26.722 INFO:tasks.workunit.client.0.smithi033.stdout:539 2024-09-21T10:06:26.723 INFO:tasks.workunit.client.0.smithi033.stdout:540 2024-09-21T10:06:26.724 INFO:tasks.workunit.client.0.smithi033.stdout:541 2024-09-21T10:06:26.725 INFO:tasks.workunit.client.0.smithi033.stdout:542 2024-09-21T10:06:26.726 INFO:tasks.workunit.client.0.smithi033.stdout:543 2024-09-21T10:06:26.727 INFO:tasks.workunit.client.0.smithi033.stdout:544 2024-09-21T10:06:26.728 INFO:tasks.workunit.client.0.smithi033.stdout:545 2024-09-21T10:06:26.729 INFO:tasks.workunit.client.0.smithi033.stdout:546 2024-09-21T10:06:26.730 INFO:tasks.workunit.client.0.smithi033.stdout:547 2024-09-21T10:06:26.731 INFO:tasks.workunit.client.0.smithi033.stdout:548 2024-09-21T10:06:26.732 INFO:tasks.workunit.client.0.smithi033.stdout:549 2024-09-21T10:06:26.733 INFO:tasks.workunit.client.0.smithi033.stdout:550 2024-09-21T10:06:26.734 INFO:tasks.workunit.client.0.smithi033.stdout:551 2024-09-21T10:06:26.735 INFO:tasks.workunit.client.0.smithi033.stdout:552 2024-09-21T10:06:26.738 INFO:tasks.workunit.client.0.smithi033.stdout:553 2024-09-21T10:06:26.738 INFO:tasks.workunit.client.0.smithi033.stdout:554 2024-09-21T10:06:26.738 INFO:tasks.workunit.client.0.smithi033.stdout:555 2024-09-21T10:06:26.739 INFO:tasks.workunit.client.0.smithi033.stdout:556 2024-09-21T10:06:26.740 INFO:tasks.workunit.client.0.smithi033.stdout:557 2024-09-21T10:06:26.741 INFO:tasks.workunit.client.0.smithi033.stdout:558 2024-09-21T10:06:26.742 INFO:tasks.workunit.client.0.smithi033.stdout:559 2024-09-21T10:06:26.743 INFO:tasks.workunit.client.0.smithi033.stdout:560 2024-09-21T10:06:26.744 INFO:tasks.workunit.client.0.smithi033.stdout:561 2024-09-21T10:06:26.745 INFO:tasks.workunit.client.0.smithi033.stdout:562 2024-09-21T10:06:26.745 INFO:tasks.workunit.client.0.smithi033.stdout:563 2024-09-21T10:06:26.746 INFO:tasks.workunit.client.0.smithi033.stdout:564 2024-09-21T10:06:26.747 INFO:tasks.workunit.client.0.smithi033.stdout:565 2024-09-21T10:06:26.749 INFO:tasks.workunit.client.0.smithi033.stdout:566 2024-09-21T10:06:26.750 INFO:tasks.workunit.client.0.smithi033.stdout:567 2024-09-21T10:06:26.751 INFO:tasks.workunit.client.0.smithi033.stdout:568 2024-09-21T10:06:26.752 INFO:tasks.workunit.client.0.smithi033.stdout:569 2024-09-21T10:06:26.753 INFO:tasks.workunit.client.0.smithi033.stdout:570 2024-09-21T10:06:26.754 INFO:tasks.workunit.client.0.smithi033.stdout:571 2024-09-21T10:06:26.755 INFO:tasks.workunit.client.0.smithi033.stdout:572 2024-09-21T10:06:26.756 INFO:tasks.workunit.client.0.smithi033.stdout:573 2024-09-21T10:06:26.757 INFO:tasks.workunit.client.0.smithi033.stdout:574 2024-09-21T10:06:26.758 INFO:tasks.workunit.client.0.smithi033.stdout:575 2024-09-21T10:06:26.759 INFO:tasks.workunit.client.0.smithi033.stdout:576 2024-09-21T10:06:26.760 INFO:tasks.workunit.client.0.smithi033.stdout:577 2024-09-21T10:06:26.761 INFO:tasks.workunit.client.0.smithi033.stdout:578 2024-09-21T10:06:26.762 INFO:tasks.workunit.client.0.smithi033.stdout:579 2024-09-21T10:06:26.763 INFO:tasks.workunit.client.0.smithi033.stdout:580 2024-09-21T10:06:26.764 INFO:tasks.workunit.client.0.smithi033.stdout:581 2024-09-21T10:06:26.765 INFO:tasks.workunit.client.0.smithi033.stdout:582 2024-09-21T10:06:26.766 INFO:tasks.workunit.client.0.smithi033.stdout:583 2024-09-21T10:06:26.767 INFO:tasks.workunit.client.0.smithi033.stdout:584 2024-09-21T10:06:26.768 INFO:tasks.workunit.client.0.smithi033.stdout:585 2024-09-21T10:06:26.769 INFO:tasks.workunit.client.0.smithi033.stdout:586 2024-09-21T10:06:26.770 INFO:tasks.workunit.client.0.smithi033.stdout:587 2024-09-21T10:06:26.771 INFO:tasks.workunit.client.0.smithi033.stdout:588 2024-09-21T10:06:26.772 INFO:tasks.workunit.client.0.smithi033.stdout:589 2024-09-21T10:06:26.773 INFO:tasks.workunit.client.0.smithi033.stdout:590 2024-09-21T10:06:26.774 INFO:tasks.workunit.client.0.smithi033.stdout:591 2024-09-21T10:06:26.775 INFO:tasks.workunit.client.0.smithi033.stdout:592 2024-09-21T10:06:26.776 INFO:tasks.workunit.client.0.smithi033.stdout:593 2024-09-21T10:06:26.777 INFO:tasks.workunit.client.0.smithi033.stdout:594 2024-09-21T10:06:26.778 INFO:tasks.workunit.client.0.smithi033.stdout:595 2024-09-21T10:06:26.779 INFO:tasks.workunit.client.0.smithi033.stdout:596 2024-09-21T10:06:26.780 INFO:tasks.workunit.client.0.smithi033.stdout:597 2024-09-21T10:06:26.781 INFO:tasks.workunit.client.0.smithi033.stdout:598 2024-09-21T10:06:26.782 INFO:tasks.workunit.client.0.smithi033.stdout:599 2024-09-21T10:06:26.783 INFO:tasks.workunit.client.0.smithi033.stdout:600 2024-09-21T10:06:26.784 INFO:tasks.workunit.client.0.smithi033.stdout:601 2024-09-21T10:06:26.785 INFO:tasks.workunit.client.0.smithi033.stdout:602 2024-09-21T10:06:26.786 INFO:tasks.workunit.client.0.smithi033.stdout:603 2024-09-21T10:06:26.787 INFO:tasks.workunit.client.0.smithi033.stdout:604 2024-09-21T10:06:26.788 INFO:tasks.workunit.client.0.smithi033.stdout:605 2024-09-21T10:06:26.789 INFO:tasks.workunit.client.0.smithi033.stdout:606 2024-09-21T10:06:26.790 INFO:tasks.workunit.client.0.smithi033.stdout:607 2024-09-21T10:06:26.791 INFO:tasks.workunit.client.0.smithi033.stdout:608 2024-09-21T10:06:26.792 INFO:tasks.workunit.client.0.smithi033.stdout:609 2024-09-21T10:06:26.793 INFO:tasks.workunit.client.0.smithi033.stdout:610 2024-09-21T10:06:26.794 INFO:tasks.workunit.client.0.smithi033.stdout:611 2024-09-21T10:06:26.795 INFO:tasks.workunit.client.0.smithi033.stdout:612 2024-09-21T10:06:26.796 INFO:tasks.workunit.client.0.smithi033.stdout:613 2024-09-21T10:06:26.797 INFO:tasks.workunit.client.0.smithi033.stdout:614 2024-09-21T10:06:26.798 INFO:tasks.workunit.client.0.smithi033.stdout:615 2024-09-21T10:06:26.799 INFO:tasks.workunit.client.0.smithi033.stdout:616 2024-09-21T10:06:26.800 INFO:tasks.workunit.client.0.smithi033.stdout:617 2024-09-21T10:06:26.801 INFO:tasks.workunit.client.0.smithi033.stdout:618 2024-09-21T10:06:26.802 INFO:tasks.workunit.client.0.smithi033.stdout:619 2024-09-21T10:06:26.803 INFO:tasks.workunit.client.0.smithi033.stdout:620 2024-09-21T10:06:26.804 INFO:tasks.workunit.client.0.smithi033.stdout:621 2024-09-21T10:06:26.805 INFO:tasks.workunit.client.0.smithi033.stdout:622 2024-09-21T10:06:26.806 INFO:tasks.workunit.client.0.smithi033.stdout:623 2024-09-21T10:06:26.806 INFO:tasks.workunit.client.0.smithi033.stdout:624 2024-09-21T10:06:26.807 INFO:tasks.workunit.client.0.smithi033.stdout:625 2024-09-21T10:06:26.808 INFO:tasks.workunit.client.0.smithi033.stdout:626 2024-09-21T10:06:26.809 INFO:tasks.workunit.client.0.smithi033.stdout:627 2024-09-21T10:06:26.810 INFO:tasks.workunit.client.0.smithi033.stdout:628 2024-09-21T10:06:26.811 INFO:tasks.workunit.client.0.smithi033.stdout:629 2024-09-21T10:06:26.812 INFO:tasks.workunit.client.0.smithi033.stdout:630 2024-09-21T10:06:26.813 INFO:tasks.workunit.client.0.smithi033.stdout:631 2024-09-21T10:06:26.814 INFO:tasks.workunit.client.0.smithi033.stdout:632 2024-09-21T10:06:26.815 INFO:tasks.workunit.client.0.smithi033.stdout:633 2024-09-21T10:06:26.817 INFO:tasks.workunit.client.0.smithi033.stdout:634 2024-09-21T10:06:26.818 INFO:tasks.workunit.client.0.smithi033.stdout:635 2024-09-21T10:06:26.819 INFO:tasks.workunit.client.0.smithi033.stdout:636 2024-09-21T10:06:26.820 INFO:tasks.workunit.client.0.smithi033.stdout:637 2024-09-21T10:06:26.821 INFO:tasks.workunit.client.0.smithi033.stdout:638 2024-09-21T10:06:26.822 INFO:tasks.workunit.client.0.smithi033.stdout:639 2024-09-21T10:06:26.823 INFO:tasks.workunit.client.0.smithi033.stdout:640 2024-09-21T10:06:26.824 INFO:tasks.workunit.client.0.smithi033.stdout:641 2024-09-21T10:06:26.825 INFO:tasks.workunit.client.0.smithi033.stdout:642 2024-09-21T10:06:26.826 INFO:tasks.workunit.client.0.smithi033.stdout:643 2024-09-21T10:06:26.827 INFO:tasks.workunit.client.0.smithi033.stdout:644 2024-09-21T10:06:26.828 INFO:tasks.workunit.client.0.smithi033.stdout:645 2024-09-21T10:06:26.829 INFO:tasks.workunit.client.0.smithi033.stdout:646 2024-09-21T10:06:26.830 INFO:tasks.workunit.client.0.smithi033.stdout:647 2024-09-21T10:06:26.831 INFO:tasks.workunit.client.0.smithi033.stdout:648 2024-09-21T10:06:26.832 INFO:tasks.workunit.client.0.smithi033.stdout:649 2024-09-21T10:06:26.833 INFO:tasks.workunit.client.0.smithi033.stdout:650 2024-09-21T10:06:26.834 INFO:tasks.workunit.client.0.smithi033.stdout:651 2024-09-21T10:06:26.835 INFO:tasks.workunit.client.0.smithi033.stdout:652 2024-09-21T10:06:26.836 INFO:tasks.workunit.client.0.smithi033.stdout:653 2024-09-21T10:06:26.837 INFO:tasks.workunit.client.0.smithi033.stdout:654 2024-09-21T10:06:26.838 INFO:tasks.workunit.client.0.smithi033.stdout:655 2024-09-21T10:06:26.839 INFO:tasks.workunit.client.0.smithi033.stdout:656 2024-09-21T10:06:26.840 INFO:tasks.workunit.client.0.smithi033.stdout:657 2024-09-21T10:06:26.841 INFO:tasks.workunit.client.0.smithi033.stdout:658 2024-09-21T10:06:26.842 INFO:tasks.workunit.client.0.smithi033.stdout:659 2024-09-21T10:06:26.844 INFO:tasks.workunit.client.0.smithi033.stdout:660 2024-09-21T10:06:26.845 INFO:tasks.workunit.client.0.smithi033.stdout:661 2024-09-21T10:06:26.846 INFO:tasks.workunit.client.0.smithi033.stdout:662 2024-09-21T10:06:26.847 INFO:tasks.workunit.client.0.smithi033.stdout:663 2024-09-21T10:06:26.848 INFO:tasks.workunit.client.0.smithi033.stdout:664 2024-09-21T10:06:26.849 INFO:tasks.workunit.client.0.smithi033.stdout:665 2024-09-21T10:06:26.850 INFO:tasks.workunit.client.0.smithi033.stdout:666 2024-09-21T10:06:26.851 INFO:tasks.workunit.client.0.smithi033.stdout:667 2024-09-21T10:06:26.852 INFO:tasks.workunit.client.0.smithi033.stdout:668 2024-09-21T10:06:26.854 INFO:tasks.workunit.client.0.smithi033.stdout:669 2024-09-21T10:06:26.855 INFO:tasks.workunit.client.0.smithi033.stdout:670 2024-09-21T10:06:26.856 INFO:tasks.workunit.client.0.smithi033.stdout:671 2024-09-21T10:06:26.857 INFO:tasks.workunit.client.0.smithi033.stdout:672 2024-09-21T10:06:26.858 INFO:tasks.workunit.client.0.smithi033.stdout:673 2024-09-21T10:06:26.859 INFO:tasks.workunit.client.0.smithi033.stdout:674 2024-09-21T10:06:26.860 INFO:tasks.workunit.client.0.smithi033.stdout:675 2024-09-21T10:06:26.861 INFO:tasks.workunit.client.0.smithi033.stdout:676 2024-09-21T10:06:26.862 INFO:tasks.workunit.client.0.smithi033.stdout:677 2024-09-21T10:06:26.863 INFO:tasks.workunit.client.0.smithi033.stdout:678 2024-09-21T10:06:26.863 INFO:tasks.workunit.client.0.smithi033.stdout:679 2024-09-21T10:06:26.865 INFO:tasks.workunit.client.0.smithi033.stdout:680 2024-09-21T10:06:26.866 INFO:tasks.workunit.client.0.smithi033.stdout:681 2024-09-21T10:06:26.867 INFO:tasks.workunit.client.0.smithi033.stdout:682 2024-09-21T10:06:26.868 INFO:tasks.workunit.client.0.smithi033.stdout:683 2024-09-21T10:06:26.869 INFO:tasks.workunit.client.0.smithi033.stdout:684 2024-09-21T10:06:26.870 INFO:tasks.workunit.client.0.smithi033.stdout:685 2024-09-21T10:06:26.871 INFO:tasks.workunit.client.0.smithi033.stdout:686 2024-09-21T10:06:26.872 INFO:tasks.workunit.client.0.smithi033.stdout:687 2024-09-21T10:06:26.873 INFO:tasks.workunit.client.0.smithi033.stdout:688 2024-09-21T10:06:26.874 INFO:tasks.workunit.client.0.smithi033.stdout:689 2024-09-21T10:06:26.875 INFO:tasks.workunit.client.0.smithi033.stdout:690 2024-09-21T10:06:26.876 INFO:tasks.workunit.client.0.smithi033.stdout:691 2024-09-21T10:06:26.877 INFO:tasks.workunit.client.0.smithi033.stdout:692 2024-09-21T10:06:26.878 INFO:tasks.workunit.client.0.smithi033.stdout:693 2024-09-21T10:06:26.879 INFO:tasks.workunit.client.0.smithi033.stdout:694 2024-09-21T10:06:26.880 INFO:tasks.workunit.client.0.smithi033.stdout:695 2024-09-21T10:06:26.881 INFO:tasks.workunit.client.0.smithi033.stdout:696 2024-09-21T10:06:26.882 INFO:tasks.workunit.client.0.smithi033.stdout:697 2024-09-21T10:06:26.883 INFO:tasks.workunit.client.0.smithi033.stdout:698 2024-09-21T10:06:26.884 INFO:tasks.workunit.client.0.smithi033.stdout:699 2024-09-21T10:06:26.885 INFO:tasks.workunit.client.0.smithi033.stdout:700 2024-09-21T10:06:26.886 INFO:tasks.workunit.client.0.smithi033.stdout:701 2024-09-21T10:06:26.887 INFO:tasks.workunit.client.0.smithi033.stdout:702 2024-09-21T10:06:26.888 INFO:tasks.workunit.client.0.smithi033.stdout:703 2024-09-21T10:06:26.889 INFO:tasks.workunit.client.0.smithi033.stdout:704 2024-09-21T10:06:26.890 INFO:tasks.workunit.client.0.smithi033.stdout:705 2024-09-21T10:06:26.891 INFO:tasks.workunit.client.0.smithi033.stdout:706 2024-09-21T10:06:26.892 INFO:tasks.workunit.client.0.smithi033.stdout:707 2024-09-21T10:06:26.893 INFO:tasks.workunit.client.0.smithi033.stdout:708 2024-09-21T10:06:26.894 INFO:tasks.workunit.client.0.smithi033.stdout:709 2024-09-21T10:06:26.895 INFO:tasks.workunit.client.0.smithi033.stdout:710 2024-09-21T10:06:26.896 INFO:tasks.workunit.client.0.smithi033.stdout:711 2024-09-21T10:06:26.897 INFO:tasks.workunit.client.0.smithi033.stdout:712 2024-09-21T10:06:26.899 INFO:tasks.workunit.client.0.smithi033.stdout:713 2024-09-21T10:06:26.900 INFO:tasks.workunit.client.0.smithi033.stdout:714 2024-09-21T10:06:26.901 INFO:tasks.workunit.client.0.smithi033.stdout:715 2024-09-21T10:06:26.902 INFO:tasks.workunit.client.0.smithi033.stdout:716 2024-09-21T10:06:26.903 INFO:tasks.workunit.client.0.smithi033.stdout:717 2024-09-21T10:06:26.904 INFO:tasks.workunit.client.0.smithi033.stdout:718 2024-09-21T10:06:26.905 INFO:tasks.workunit.client.0.smithi033.stdout:719 2024-09-21T10:06:26.907 INFO:tasks.workunit.client.0.smithi033.stdout:720 2024-09-21T10:06:26.908 INFO:tasks.workunit.client.0.smithi033.stdout:721 2024-09-21T10:06:26.909 INFO:tasks.workunit.client.0.smithi033.stdout:722 2024-09-21T10:06:26.910 INFO:tasks.workunit.client.0.smithi033.stdout:723 2024-09-21T10:06:26.911 INFO:tasks.workunit.client.0.smithi033.stdout:724 2024-09-21T10:06:26.912 INFO:tasks.workunit.client.0.smithi033.stdout:725 2024-09-21T10:06:26.913 INFO:tasks.workunit.client.0.smithi033.stdout:726 2024-09-21T10:06:26.914 INFO:tasks.workunit.client.0.smithi033.stdout:727 2024-09-21T10:06:26.915 INFO:tasks.workunit.client.0.smithi033.stdout:728 2024-09-21T10:06:26.916 INFO:tasks.workunit.client.0.smithi033.stdout:729 2024-09-21T10:06:26.917 INFO:tasks.workunit.client.0.smithi033.stdout:730 2024-09-21T10:06:26.918 INFO:tasks.workunit.client.0.smithi033.stdout:731 2024-09-21T10:06:26.919 INFO:tasks.workunit.client.0.smithi033.stdout:732 2024-09-21T10:06:26.920 INFO:tasks.workunit.client.0.smithi033.stdout:733 2024-09-21T10:06:26.921 INFO:tasks.workunit.client.0.smithi033.stdout:734 2024-09-21T10:06:26.923 INFO:tasks.workunit.client.0.smithi033.stdout:735 2024-09-21T10:06:26.924 INFO:tasks.workunit.client.0.smithi033.stdout:736 2024-09-21T10:06:26.925 INFO:tasks.workunit.client.0.smithi033.stdout:737 2024-09-21T10:06:26.926 INFO:tasks.workunit.client.0.smithi033.stdout:738 2024-09-21T10:06:26.927 INFO:tasks.workunit.client.0.smithi033.stdout:739 2024-09-21T10:06:26.928 INFO:tasks.workunit.client.0.smithi033.stdout:740 2024-09-21T10:06:26.929 INFO:tasks.workunit.client.0.smithi033.stdout:741 2024-09-21T10:06:26.930 INFO:tasks.workunit.client.0.smithi033.stdout:742 2024-09-21T10:06:26.931 INFO:tasks.workunit.client.0.smithi033.stdout:743 2024-09-21T10:06:26.932 INFO:tasks.workunit.client.0.smithi033.stdout:744 2024-09-21T10:06:26.933 INFO:tasks.workunit.client.0.smithi033.stdout:745 2024-09-21T10:06:26.934 INFO:tasks.workunit.client.0.smithi033.stdout:746 2024-09-21T10:06:26.935 INFO:tasks.workunit.client.0.smithi033.stdout:747 2024-09-21T10:06:26.936 INFO:tasks.workunit.client.0.smithi033.stdout:748 2024-09-21T10:06:26.937 INFO:tasks.workunit.client.0.smithi033.stdout:749 2024-09-21T10:06:26.938 INFO:tasks.workunit.client.0.smithi033.stdout:750 2024-09-21T10:06:26.939 INFO:tasks.workunit.client.0.smithi033.stdout:751 2024-09-21T10:06:26.941 INFO:tasks.workunit.client.0.smithi033.stdout:752 2024-09-21T10:06:26.942 INFO:tasks.workunit.client.0.smithi033.stdout:753 2024-09-21T10:06:26.942 INFO:tasks.workunit.client.0.smithi033.stdout:754 2024-09-21T10:06:26.944 INFO:tasks.workunit.client.0.smithi033.stdout:755 2024-09-21T10:06:26.945 INFO:tasks.workunit.client.0.smithi033.stdout:756 2024-09-21T10:06:26.946 INFO:tasks.workunit.client.0.smithi033.stdout:757 2024-09-21T10:06:26.947 INFO:tasks.workunit.client.0.smithi033.stdout:758 2024-09-21T10:06:26.948 INFO:tasks.workunit.client.0.smithi033.stdout:759 2024-09-21T10:06:26.949 INFO:tasks.workunit.client.0.smithi033.stdout:760 2024-09-21T10:06:26.950 INFO:tasks.workunit.client.0.smithi033.stdout:761 2024-09-21T10:06:26.951 INFO:tasks.workunit.client.0.smithi033.stdout:762 2024-09-21T10:06:26.952 INFO:tasks.workunit.client.0.smithi033.stdout:763 2024-09-21T10:06:26.954 INFO:tasks.workunit.client.0.smithi033.stdout:764 2024-09-21T10:06:26.955 INFO:tasks.workunit.client.0.smithi033.stdout:765 2024-09-21T10:06:26.956 INFO:tasks.workunit.client.0.smithi033.stdout:766 2024-09-21T10:06:26.957 INFO:tasks.workunit.client.0.smithi033.stdout:767 2024-09-21T10:06:26.958 INFO:tasks.workunit.client.0.smithi033.stdout:768 2024-09-21T10:06:26.959 INFO:tasks.workunit.client.0.smithi033.stdout:769 2024-09-21T10:06:26.960 INFO:tasks.workunit.client.0.smithi033.stdout:770 2024-09-21T10:06:26.961 INFO:tasks.workunit.client.0.smithi033.stdout:771 2024-09-21T10:06:26.962 INFO:tasks.workunit.client.0.smithi033.stdout:772 2024-09-21T10:06:26.963 INFO:tasks.workunit.client.0.smithi033.stdout:773 2024-09-21T10:06:26.964 INFO:tasks.workunit.client.0.smithi033.stdout:774 2024-09-21T10:06:26.965 INFO:tasks.workunit.client.0.smithi033.stdout:775 2024-09-21T10:06:26.967 INFO:tasks.workunit.client.0.smithi033.stdout:776 2024-09-21T10:06:26.968 INFO:tasks.workunit.client.0.smithi033.stdout:777 2024-09-21T10:06:26.969 INFO:tasks.workunit.client.0.smithi033.stdout:778 2024-09-21T10:06:26.970 INFO:tasks.workunit.client.0.smithi033.stdout:779 2024-09-21T10:06:26.971 INFO:tasks.workunit.client.0.smithi033.stdout:780 2024-09-21T10:06:26.972 INFO:tasks.workunit.client.0.smithi033.stdout:781 2024-09-21T10:06:26.973 INFO:tasks.workunit.client.0.smithi033.stdout:782 2024-09-21T10:06:26.974 INFO:tasks.workunit.client.0.smithi033.stdout:783 2024-09-21T10:06:26.975 INFO:tasks.workunit.client.0.smithi033.stdout:784 2024-09-21T10:06:26.976 INFO:tasks.workunit.client.0.smithi033.stdout:785 2024-09-21T10:06:26.977 INFO:tasks.workunit.client.0.smithi033.stdout:786 2024-09-21T10:06:26.978 INFO:tasks.workunit.client.0.smithi033.stdout:787 2024-09-21T10:06:26.979 INFO:tasks.workunit.client.0.smithi033.stdout:788 2024-09-21T10:06:26.980 INFO:tasks.workunit.client.0.smithi033.stdout:789 2024-09-21T10:06:26.981 INFO:tasks.workunit.client.0.smithi033.stdout:790 2024-09-21T10:06:26.983 INFO:tasks.workunit.client.0.smithi033.stdout:791 2024-09-21T10:06:26.984 INFO:tasks.workunit.client.0.smithi033.stdout:792 2024-09-21T10:06:26.985 INFO:tasks.workunit.client.0.smithi033.stdout:793 2024-09-21T10:06:26.986 INFO:tasks.workunit.client.0.smithi033.stdout:794 2024-09-21T10:06:26.987 INFO:tasks.workunit.client.0.smithi033.stdout:795 2024-09-21T10:06:26.988 INFO:tasks.workunit.client.0.smithi033.stdout:796 2024-09-21T10:06:26.989 INFO:tasks.workunit.client.0.smithi033.stdout:797 2024-09-21T10:06:26.990 INFO:tasks.workunit.client.0.smithi033.stdout:798 2024-09-21T10:06:26.991 INFO:tasks.workunit.client.0.smithi033.stdout:799 2024-09-21T10:06:26.992 INFO:tasks.workunit.client.0.smithi033.stdout:800 2024-09-21T10:06:26.993 INFO:tasks.workunit.client.0.smithi033.stdout:801 2024-09-21T10:06:26.994 INFO:tasks.workunit.client.0.smithi033.stdout:802 2024-09-21T10:06:26.995 INFO:tasks.workunit.client.0.smithi033.stdout:803 2024-09-21T10:06:26.997 INFO:tasks.workunit.client.0.smithi033.stdout:804 2024-09-21T10:06:26.998 INFO:tasks.workunit.client.0.smithi033.stdout:805 2024-09-21T10:06:26.999 INFO:tasks.workunit.client.0.smithi033.stdout:806 2024-09-21T10:06:27.000 INFO:tasks.workunit.client.0.smithi033.stdout:807 2024-09-21T10:06:27.001 INFO:tasks.workunit.client.0.smithi033.stdout:808 2024-09-21T10:06:27.002 INFO:tasks.workunit.client.0.smithi033.stdout:809 2024-09-21T10:06:27.003 INFO:tasks.workunit.client.0.smithi033.stdout:810 2024-09-21T10:06:27.004 INFO:tasks.workunit.client.0.smithi033.stdout:811 2024-09-21T10:06:27.005 INFO:tasks.workunit.client.0.smithi033.stdout:812 2024-09-21T10:06:27.006 INFO:tasks.workunit.client.0.smithi033.stdout:813 2024-09-21T10:06:27.007 INFO:tasks.workunit.client.0.smithi033.stdout:814 2024-09-21T10:06:27.008 INFO:tasks.workunit.client.0.smithi033.stdout:815 2024-09-21T10:06:27.009 INFO:tasks.workunit.client.0.smithi033.stdout:816 2024-09-21T10:06:27.010 INFO:tasks.workunit.client.0.smithi033.stdout:817 2024-09-21T10:06:27.011 INFO:tasks.workunit.client.0.smithi033.stdout:818 2024-09-21T10:06:27.012 INFO:tasks.workunit.client.0.smithi033.stdout:819 2024-09-21T10:06:27.014 INFO:tasks.workunit.client.0.smithi033.stdout:820 2024-09-21T10:06:27.015 INFO:tasks.workunit.client.0.smithi033.stdout:821 2024-09-21T10:06:27.016 INFO:tasks.workunit.client.0.smithi033.stdout:822 2024-09-21T10:06:27.017 INFO:tasks.workunit.client.0.smithi033.stdout:823 2024-09-21T10:06:27.018 INFO:tasks.workunit.client.0.smithi033.stdout:824 2024-09-21T10:06:27.019 INFO:tasks.workunit.client.0.smithi033.stdout:825 2024-09-21T10:06:27.020 INFO:tasks.workunit.client.0.smithi033.stdout:826 2024-09-21T10:06:27.021 INFO:tasks.workunit.client.0.smithi033.stdout:827 2024-09-21T10:06:27.022 INFO:tasks.workunit.client.0.smithi033.stdout:828 2024-09-21T10:06:27.023 INFO:tasks.workunit.client.0.smithi033.stdout:829 2024-09-21T10:06:27.024 INFO:tasks.workunit.client.0.smithi033.stdout:830 2024-09-21T10:06:27.025 INFO:tasks.workunit.client.0.smithi033.stdout:831 2024-09-21T10:06:27.026 INFO:tasks.workunit.client.0.smithi033.stdout:832 2024-09-21T10:06:27.027 INFO:tasks.workunit.client.0.smithi033.stdout:833 2024-09-21T10:06:27.028 INFO:tasks.workunit.client.0.smithi033.stdout:834 2024-09-21T10:06:27.029 INFO:tasks.workunit.client.0.smithi033.stdout:835 2024-09-21T10:06:27.030 INFO:tasks.workunit.client.0.smithi033.stdout:836 2024-09-21T10:06:27.031 INFO:tasks.workunit.client.0.smithi033.stdout:837 2024-09-21T10:06:27.032 INFO:tasks.workunit.client.0.smithi033.stdout:838 2024-09-21T10:06:27.033 INFO:tasks.workunit.client.0.smithi033.stdout:839 2024-09-21T10:06:27.034 INFO:tasks.workunit.client.0.smithi033.stdout:840 2024-09-21T10:06:27.035 INFO:tasks.workunit.client.0.smithi033.stdout:841 2024-09-21T10:06:27.036 INFO:tasks.workunit.client.0.smithi033.stdout:842 2024-09-21T10:06:27.037 INFO:tasks.workunit.client.0.smithi033.stdout:843 2024-09-21T10:06:27.039 INFO:tasks.workunit.client.0.smithi033.stdout:844 2024-09-21T10:06:27.040 INFO:tasks.workunit.client.0.smithi033.stdout:845 2024-09-21T10:06:27.041 INFO:tasks.workunit.client.0.smithi033.stdout:846 2024-09-21T10:06:27.042 INFO:tasks.workunit.client.0.smithi033.stdout:847 2024-09-21T10:06:27.043 INFO:tasks.workunit.client.0.smithi033.stdout:848 2024-09-21T10:06:27.044 INFO:tasks.workunit.client.0.smithi033.stdout:849 2024-09-21T10:06:27.045 INFO:tasks.workunit.client.0.smithi033.stdout:850 2024-09-21T10:06:27.046 INFO:tasks.workunit.client.0.smithi033.stdout:851 2024-09-21T10:06:27.047 INFO:tasks.workunit.client.0.smithi033.stdout:852 2024-09-21T10:06:27.049 INFO:tasks.workunit.client.0.smithi033.stdout:853 2024-09-21T10:06:27.050 INFO:tasks.workunit.client.0.smithi033.stdout:854 2024-09-21T10:06:27.051 INFO:tasks.workunit.client.0.smithi033.stdout:855 2024-09-21T10:06:27.052 INFO:tasks.workunit.client.0.smithi033.stdout:856 2024-09-21T10:06:27.053 INFO:tasks.workunit.client.0.smithi033.stdout:857 2024-09-21T10:06:27.054 INFO:tasks.workunit.client.0.smithi033.stdout:858 2024-09-21T10:06:27.055 INFO:tasks.workunit.client.0.smithi033.stdout:859 2024-09-21T10:06:27.056 INFO:tasks.workunit.client.0.smithi033.stdout:860 2024-09-21T10:06:27.057 INFO:tasks.workunit.client.0.smithi033.stdout:861 2024-09-21T10:06:27.058 INFO:tasks.workunit.client.0.smithi033.stdout:862 2024-09-21T10:06:27.059 INFO:tasks.workunit.client.0.smithi033.stdout:863 2024-09-21T10:06:27.060 INFO:tasks.workunit.client.0.smithi033.stdout:864 2024-09-21T10:06:27.061 INFO:tasks.workunit.client.0.smithi033.stdout:865 2024-09-21T10:06:27.062 INFO:tasks.workunit.client.0.smithi033.stdout:866 2024-09-21T10:06:27.063 INFO:tasks.workunit.client.0.smithi033.stdout:867 2024-09-21T10:06:27.064 INFO:tasks.workunit.client.0.smithi033.stdout:868 2024-09-21T10:06:27.065 INFO:tasks.workunit.client.0.smithi033.stdout:869 2024-09-21T10:06:27.066 INFO:tasks.workunit.client.0.smithi033.stdout:870 2024-09-21T10:06:27.067 INFO:tasks.workunit.client.0.smithi033.stdout:871 2024-09-21T10:06:27.069 INFO:tasks.workunit.client.0.smithi033.stdout:872 2024-09-21T10:06:27.070 INFO:tasks.workunit.client.0.smithi033.stdout:873 2024-09-21T10:06:27.071 INFO:tasks.workunit.client.0.smithi033.stdout:874 2024-09-21T10:06:27.072 INFO:tasks.workunit.client.0.smithi033.stdout:875 2024-09-21T10:06:27.073 INFO:tasks.workunit.client.0.smithi033.stdout:876 2024-09-21T10:06:27.074 INFO:tasks.workunit.client.0.smithi033.stdout:877 2024-09-21T10:06:27.075 INFO:tasks.workunit.client.0.smithi033.stdout:878 2024-09-21T10:06:27.076 INFO:tasks.workunit.client.0.smithi033.stdout:879 2024-09-21T10:06:27.077 INFO:tasks.workunit.client.0.smithi033.stdout:880 2024-09-21T10:06:27.078 INFO:tasks.workunit.client.0.smithi033.stdout:881 2024-09-21T10:06:27.079 INFO:tasks.workunit.client.0.smithi033.stdout:882 2024-09-21T10:06:27.080 INFO:tasks.workunit.client.0.smithi033.stdout:883 2024-09-21T10:06:27.082 INFO:tasks.workunit.client.0.smithi033.stdout:884 2024-09-21T10:06:27.083 INFO:tasks.workunit.client.0.smithi033.stdout:885 2024-09-21T10:06:27.084 INFO:tasks.workunit.client.0.smithi033.stdout:886 2024-09-21T10:06:27.085 INFO:tasks.workunit.client.0.smithi033.stdout:887 2024-09-21T10:06:27.086 INFO:tasks.workunit.client.0.smithi033.stdout:888 2024-09-21T10:06:27.087 INFO:tasks.workunit.client.0.smithi033.stdout:889 2024-09-21T10:06:27.088 INFO:tasks.workunit.client.0.smithi033.stdout:890 2024-09-21T10:06:27.089 INFO:tasks.workunit.client.0.smithi033.stdout:891 2024-09-21T10:06:27.091 INFO:tasks.workunit.client.0.smithi033.stdout:892 2024-09-21T10:06:27.092 INFO:tasks.workunit.client.0.smithi033.stdout:893 2024-09-21T10:06:27.093 INFO:tasks.workunit.client.0.smithi033.stdout:894 2024-09-21T10:06:27.094 INFO:tasks.workunit.client.0.smithi033.stdout:895 2024-09-21T10:06:27.095 INFO:tasks.workunit.client.0.smithi033.stdout:896 2024-09-21T10:06:27.097 INFO:tasks.workunit.client.0.smithi033.stdout:897 2024-09-21T10:06:27.098 INFO:tasks.workunit.client.0.smithi033.stdout:898 2024-09-21T10:06:27.099 INFO:tasks.workunit.client.0.smithi033.stdout:899 2024-09-21T10:06:27.100 INFO:tasks.workunit.client.0.smithi033.stdout:900 2024-09-21T10:06:27.101 INFO:tasks.workunit.client.0.smithi033.stdout:901 2024-09-21T10:06:27.102 INFO:tasks.workunit.client.0.smithi033.stdout:902 2024-09-21T10:06:27.103 INFO:tasks.workunit.client.0.smithi033.stdout:903 2024-09-21T10:06:27.104 INFO:tasks.workunit.client.0.smithi033.stdout:904 2024-09-21T10:06:27.105 INFO:tasks.workunit.client.0.smithi033.stdout:905 2024-09-21T10:06:27.106 INFO:tasks.workunit.client.0.smithi033.stdout:906 2024-09-21T10:06:27.108 INFO:tasks.workunit.client.0.smithi033.stdout:907 2024-09-21T10:06:27.109 INFO:tasks.workunit.client.0.smithi033.stdout:908 2024-09-21T10:06:27.110 INFO:tasks.workunit.client.0.smithi033.stdout:909 2024-09-21T10:06:27.111 INFO:tasks.workunit.client.0.smithi033.stdout:910 2024-09-21T10:06:27.112 INFO:tasks.workunit.client.0.smithi033.stdout:911 2024-09-21T10:06:27.113 INFO:tasks.workunit.client.0.smithi033.stdout:912 2024-09-21T10:06:27.114 INFO:tasks.workunit.client.0.smithi033.stdout:913 2024-09-21T10:06:27.116 INFO:tasks.workunit.client.0.smithi033.stdout:914 2024-09-21T10:06:27.117 INFO:tasks.workunit.client.0.smithi033.stdout:915 2024-09-21T10:06:27.118 INFO:tasks.workunit.client.0.smithi033.stdout:916 2024-09-21T10:06:27.119 INFO:tasks.workunit.client.0.smithi033.stdout:917 2024-09-21T10:06:27.120 INFO:tasks.workunit.client.0.smithi033.stdout:918 2024-09-21T10:06:27.121 INFO:tasks.workunit.client.0.smithi033.stdout:919 2024-09-21T10:06:27.122 INFO:tasks.workunit.client.0.smithi033.stdout:920 2024-09-21T10:06:27.123 INFO:tasks.workunit.client.0.smithi033.stdout:921 2024-09-21T10:06:27.124 INFO:tasks.workunit.client.0.smithi033.stdout:922 2024-09-21T10:06:27.125 INFO:tasks.workunit.client.0.smithi033.stdout:923 2024-09-21T10:06:27.127 INFO:tasks.workunit.client.0.smithi033.stdout:924 2024-09-21T10:06:27.128 INFO:tasks.workunit.client.0.smithi033.stdout:925 2024-09-21T10:06:27.129 INFO:tasks.workunit.client.0.smithi033.stdout:926 2024-09-21T10:06:27.130 INFO:tasks.workunit.client.0.smithi033.stdout:927 2024-09-21T10:06:27.131 INFO:tasks.workunit.client.0.smithi033.stdout:928 2024-09-21T10:06:27.132 INFO:tasks.workunit.client.0.smithi033.stdout:929 2024-09-21T10:06:27.133 INFO:tasks.workunit.client.0.smithi033.stdout:930 2024-09-21T10:06:27.134 INFO:tasks.workunit.client.0.smithi033.stdout:931 2024-09-21T10:06:27.136 INFO:tasks.workunit.client.0.smithi033.stdout:932 2024-09-21T10:06:27.137 INFO:tasks.workunit.client.0.smithi033.stdout:933 2024-09-21T10:06:27.138 INFO:tasks.workunit.client.0.smithi033.stdout:934 2024-09-21T10:06:27.139 INFO:tasks.workunit.client.0.smithi033.stdout:935 2024-09-21T10:06:27.141 INFO:tasks.workunit.client.0.smithi033.stdout:936 2024-09-21T10:06:27.142 INFO:tasks.workunit.client.0.smithi033.stdout:937 2024-09-21T10:06:27.143 INFO:tasks.workunit.client.0.smithi033.stdout:938 2024-09-21T10:06:27.145 INFO:tasks.workunit.client.0.smithi033.stdout:939 2024-09-21T10:06:27.145 INFO:tasks.workunit.client.0.smithi033.stdout:940 2024-09-21T10:06:27.146 INFO:tasks.workunit.client.0.smithi033.stdout:941 2024-09-21T10:06:27.148 INFO:tasks.workunit.client.0.smithi033.stdout:942 2024-09-21T10:06:27.149 INFO:tasks.workunit.client.0.smithi033.stdout:943 2024-09-21T10:06:27.150 INFO:tasks.workunit.client.0.smithi033.stdout:944 2024-09-21T10:06:27.151 INFO:tasks.workunit.client.0.smithi033.stdout:945 2024-09-21T10:06:27.153 INFO:tasks.workunit.client.0.smithi033.stdout:946 2024-09-21T10:06:27.154 INFO:tasks.workunit.client.0.smithi033.stdout:947 2024-09-21T10:06:27.155 INFO:tasks.workunit.client.0.smithi033.stdout:948 2024-09-21T10:06:27.156 INFO:tasks.workunit.client.0.smithi033.stdout:949 2024-09-21T10:06:27.157 INFO:tasks.workunit.client.0.smithi033.stdout:950 2024-09-21T10:06:27.159 INFO:tasks.workunit.client.0.smithi033.stdout:951 2024-09-21T10:06:27.160 INFO:tasks.workunit.client.0.smithi033.stdout:952 2024-09-21T10:06:27.161 INFO:tasks.workunit.client.0.smithi033.stdout:953 2024-09-21T10:06:27.162 INFO:tasks.workunit.client.0.smithi033.stdout:954 2024-09-21T10:06:27.163 INFO:tasks.workunit.client.0.smithi033.stdout:955 2024-09-21T10:06:27.165 INFO:tasks.workunit.client.0.smithi033.stdout:956 2024-09-21T10:06:27.166 INFO:tasks.workunit.client.0.smithi033.stdout:957 2024-09-21T10:06:27.167 INFO:tasks.workunit.client.0.smithi033.stdout:958 2024-09-21T10:06:27.168 INFO:tasks.workunit.client.0.smithi033.stdout:959 2024-09-21T10:06:27.169 INFO:tasks.workunit.client.0.smithi033.stdout:960 2024-09-21T10:06:27.171 INFO:tasks.workunit.client.0.smithi033.stdout:961 2024-09-21T10:06:27.172 INFO:tasks.workunit.client.0.smithi033.stdout:962 2024-09-21T10:06:27.173 INFO:tasks.workunit.client.0.smithi033.stdout:963 2024-09-21T10:06:27.174 INFO:tasks.workunit.client.0.smithi033.stdout:964 2024-09-21T10:06:27.176 INFO:tasks.workunit.client.0.smithi033.stdout:965 2024-09-21T10:06:27.177 INFO:tasks.workunit.client.0.smithi033.stdout:966 2024-09-21T10:06:27.178 INFO:tasks.workunit.client.0.smithi033.stdout:967 2024-09-21T10:06:27.180 INFO:tasks.workunit.client.0.smithi033.stdout:968 2024-09-21T10:06:27.181 INFO:tasks.workunit.client.0.smithi033.stdout:969 2024-09-21T10:06:27.182 INFO:tasks.workunit.client.0.smithi033.stdout:970 2024-09-21T10:06:27.183 INFO:tasks.workunit.client.0.smithi033.stdout:971 2024-09-21T10:06:27.184 INFO:tasks.workunit.client.0.smithi033.stdout:972 2024-09-21T10:06:27.185 INFO:tasks.workunit.client.0.smithi033.stdout:973 2024-09-21T10:06:27.187 INFO:tasks.workunit.client.0.smithi033.stdout:974 2024-09-21T10:06:27.188 INFO:tasks.workunit.client.0.smithi033.stdout:975 2024-09-21T10:06:27.191 INFO:tasks.workunit.client.0.smithi033.stdout:976 2024-09-21T10:06:27.196 INFO:tasks.workunit.client.0.smithi033.stdout:977 2024-09-21T10:06:27.197 INFO:tasks.workunit.client.0.smithi033.stdout:978 2024-09-21T10:06:27.198 INFO:tasks.workunit.client.0.smithi033.stdout:979 2024-09-21T10:06:27.199 INFO:tasks.workunit.client.0.smithi033.stdout:980 2024-09-21T10:06:27.201 INFO:tasks.workunit.client.0.smithi033.stdout:981 2024-09-21T10:06:27.202 INFO:tasks.workunit.client.0.smithi033.stdout:982 2024-09-21T10:06:27.203 INFO:tasks.workunit.client.0.smithi033.stdout:983 2024-09-21T10:06:27.204 INFO:tasks.workunit.client.0.smithi033.stdout:984 2024-09-21T10:06:27.205 INFO:tasks.workunit.client.0.smithi033.stdout:985 2024-09-21T10:06:27.206 INFO:tasks.workunit.client.0.smithi033.stdout:986 2024-09-21T10:06:27.208 INFO:tasks.workunit.client.0.smithi033.stdout:987 2024-09-21T10:06:27.209 INFO:tasks.workunit.client.0.smithi033.stdout:988 2024-09-21T10:06:27.210 INFO:tasks.workunit.client.0.smithi033.stdout:989 2024-09-21T10:06:27.211 INFO:tasks.workunit.client.0.smithi033.stdout:990 2024-09-21T10:06:27.213 INFO:tasks.workunit.client.0.smithi033.stdout:991 2024-09-21T10:06:27.214 INFO:tasks.workunit.client.0.smithi033.stdout:992 2024-09-21T10:06:27.215 INFO:tasks.workunit.client.0.smithi033.stdout:993 2024-09-21T10:06:27.216 INFO:tasks.workunit.client.0.smithi033.stdout:994 2024-09-21T10:06:27.217 INFO:tasks.workunit.client.0.smithi033.stdout:995 2024-09-21T10:06:27.218 INFO:tasks.workunit.client.0.smithi033.stdout:996 2024-09-21T10:06:27.220 INFO:tasks.workunit.client.0.smithi033.stdout:997 2024-09-21T10:06:27.221 INFO:tasks.workunit.client.0.smithi033.stdout:998 2024-09-21T10:06:27.222 INFO:tasks.workunit.client.0.smithi033.stdout:999 2024-09-21T10:06:27.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:27 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:27.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:27 smithi033 ceph-mon[98546]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T10:06:27.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:27 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/299527596' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-21T10:06:27.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:27 smithi033 ceph-mon[98546]: pgmap v1011: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:27.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:27 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:27.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:27 smithi033 ceph-mon[96568]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T10:06:27.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:27 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/299527596' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-21T10:06:27.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:27 smithi033 ceph-mon[96568]: pgmap v1011: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:27.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:27 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:27.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:27 smithi105 ceph-mon[86649]: osdmap e902: 8 total, 8 up, 8 in 2024-09-21T10:06:27.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:27 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/299527596' entity='client.admin' cmd={"prefix": "osd dump","format":"json"} : dispatch 2024-09-21T10:06:27.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:27 smithi105 ceph-mon[86649]: pgmap v1011: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:28.205 INFO:tasks.workunit.client.0.smithi033.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-21T10:06:28.205 INFO:tasks.workunit.client.0.smithi033.stdout:00000010 69 73 |is| 2024-09-21T10:06:28.205 INFO:tasks.workunit.client.0.smithi033.stdout:00000012 2024-09-21T10:06:28.205 INFO:tasks.workunit.client.0.smithi033.stdout:00000000 79 6f 75 20 6d 69 67 68 74 20 73 65 65 20 74 68 |you might see th| 2024-09-21T10:06:28.205 INFO:tasks.workunit.client.0.smithi033.stdout:00000010 69 73 |is| 2024-09-21T10:06:28.205 INFO:tasks.workunit.client.0.smithi033.stdout:00000012 2024-09-21T10:06:28.205 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsHello.WriteReturnData (4041 ms) 2024-09-21T10:06:28.205 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsHello.Loud 2024-09-21T10:06:28.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:28 smithi033 ceph-mon[98546]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T10:06:28.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:28 smithi033 ceph-mon[96568]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T10:06:28.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:28 smithi105 ceph-mon[86649]: osdmap e903: 8 total, 8 up, 8 in 2024-09-21T10:06:29.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:29 smithi033 ceph-mon[98546]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T10:06:29.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:29 smithi033 ceph-mon[98546]: pgmap v1014: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:29.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:29 smithi033 ceph-mon[96568]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T10:06:29.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:29 smithi033 ceph-mon[96568]: pgmap v1014: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:29.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:29 smithi105 ceph-mon[86649]: osdmap e904: 8 total, 8 up, 8 in 2024-09-21T10:06:29.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:29 smithi105 ceph-mon[86649]: pgmap v1014: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:30.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:30 smithi033 ceph-mon[96568]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T10:06:30.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:30 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2781117581' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:30.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:30 smithi033 ceph-mon[98546]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T10:06:30.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:30 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2781117581' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:30.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:30 smithi105 ceph-mon[86649]: osdmap e905: 8 total, 8 up, 8 in 2024-09-21T10:06:30.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:30 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2781117581' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:30.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:06:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:06:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:31.242 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsHello.Loud (3036 ms) 2024-09-21T10:06:31.242 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsHello.BadMethods 2024-09-21T10:06:31.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2781117581' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:31.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-mon[98546]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T10:06:31.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-mon[98546]: pgmap v1017: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:31.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:31.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2781117581' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:31.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-mon[96568]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T10:06:31.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-mon[96568]: pgmap v1017: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:31.497 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:31.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:31 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2781117581' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:31.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:31 smithi105 ceph-mon[86649]: osdmap e906: 8 total, 8 up, 8 in 2024-09-21T10:06:31.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:31 smithi105 ceph-mon[86649]: pgmap v1017: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:31.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:06:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:06:31] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-21T10:06:32.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:32 smithi033 ceph-mon[98546]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T10:06:32.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:32 smithi033 ceph-mon[96568]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T10:06:32.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:32 smithi105 ceph-mon[86649]: osdmap e907: 8 total, 8 up, 8 in 2024-09-21T10:06:33.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:33 smithi105 ceph-mon[86649]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T10:06:33.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:33 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/184238212' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:33.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:33 smithi105 ceph-mon[86649]: pgmap v1020: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:33.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:33 smithi033 ceph-mon[96568]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T10:06:33.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:33 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/184238212' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:33.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:33 smithi033 ceph-mon[96568]: pgmap v1020: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:33.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:33 smithi033 ceph-mon[98546]: osdmap e908: 8 total, 8 up, 8 in 2024-09-21T10:06:33.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:33 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/184238212' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:33.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:33 smithi033 ceph-mon[98546]: pgmap v1020: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:33.747 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:06:33 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T10:06:33.270+0000 7f6334a30640 -1 osd.3 pg_epoch: 909 pg[38.2( v 909'1 (0'0,909'1] local-lis/les=908/909 n=1 ec=908/908 lis/c=908/908 les/c/f=909/909/0 sis=908) [3,7,5] r=0 lpr=908 crt=909'1 lcod 0'0 mlcod 0'0 active+clean] method hello.bad_reader tried to read object but is not marked RD 2024-09-21T10:06:33.747 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:06:33 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T10:06:33.271+0000 7f6330a28640 -1 osd.3 pg_epoch: 909 pg[38.2( v 909'2 (0'0,909'2] local-lis/les=908/909 n=1 ec=908/908 lis/c=908/908 les/c/f=909/909/0 sis=908) [3,7,5] r=0 lpr=908 crt=909'1 lcod 909'1 mlcod 909'1 active+clean] method hello.bad_writer tried to update object but is not marked WR 2024-09-21T10:06:34.257 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsHello.BadMethods (3016 ms) 2024-09-21T10:06:34.257 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsHello.Filter 2024-09-21T10:06:34.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:34 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:34.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:34 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/184238212' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:34.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:34 smithi105 ceph-mon[86649]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T10:06:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:34 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:34 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/184238212' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:34.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:34 smithi033 ceph-mon[98546]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T10:06:34.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:34 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:34.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:34 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/184238212' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:34.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:34 smithi033 ceph-mon[96568]: osdmap e909: 8 total, 8 up, 8 in 2024-09-21T10:06:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:35 smithi105 ceph-mon[86649]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T10:06:35.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:35 smithi105 ceph-mon[86649]: pgmap v1023: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:35 smithi033 ceph-mon[98546]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T10:06:35.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:35 smithi033 ceph-mon[98546]: pgmap v1023: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:35 smithi033 ceph-mon[96568]: osdmap e910: 8 total, 8 up, 8 in 2024-09-21T10:06:35.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:35 smithi033 ceph-mon[96568]: pgmap v1023: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:36.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:36 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:36.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:36 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:36.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:36 smithi105 ceph-mon[86649]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T10:06:36.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:36 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3512463531' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:36 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:36 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:36 smithi033 ceph-mon[98546]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T10:06:36.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:36 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3512463531' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:36.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:36 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:36.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:36 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:36.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:36 smithi033 ceph-mon[96568]: osdmap e911: 8 total, 8 up, 8 in 2024-09-21T10:06:36.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:36 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3512463531' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:37.286 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsHello.Filter (3029 ms) 2024-09-21T10:06:37.286 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 6 tests from ClsHello (19013 ms total) 2024-09-21T10:06:37.286 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:06:37.286 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:06:37.286 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 6 tests from 1 test suite ran. (19013 ms total) 2024-09-21T10:06:37.286 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 6 tests. 2024-09-21T10:06:37.288 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T10:06:37.289 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:06:37.359 INFO:tasks.workunit:Running workunit cls/test_cls_journal.sh... 2024-09-21T10:06:37.360 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_journal.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_journal.sh 2024-09-21T10:06:37.431 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:06:37.432 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 27 tests from 1 test suite. 2024-09-21T10:06:37.432 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:06:37.432 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 27 tests from TestClsJournal 2024-09-21T10:06:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:37 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3512463531' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:37 smithi033 ceph-mon[96568]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T10:06:37.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:37 smithi033 ceph-mon[96568]: pgmap v1026: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:37.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:37 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3512463531' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:37.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:37 smithi033 ceph-mon[98546]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T10:06:37.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:37 smithi033 ceph-mon[98546]: pgmap v1026: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:37 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3512463531' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129279-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:37 smithi105 ceph-mon[86649]: osdmap e912: 8 total, 8 up, 8 in 2024-09-21T10:06:37.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:37 smithi105 ceph-mon[86649]: pgmap v1026: 137 pgs: 6 creating+peering, 26 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:38.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:38 smithi105 ceph-mon[86649]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T10:06:38.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:38 smithi033 ceph-mon[96568]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T10:06:38.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:38 smithi033 ceph-mon[98546]: osdmap e913: 8 total, 8 up, 8 in 2024-09-21T10:06:39.298 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.Create 2024-09-21T10:06:39.315 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.Create (17 ms) 2024-09-21T10:06:39.315 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.MinimumSet 2024-09-21T10:06:39.320 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.MinimumSet (5 ms) 2024-09-21T10:06:39.320 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.MinimumSetStale 2024-09-21T10:06:39.326 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.MinimumSetStale (6 ms) 2024-09-21T10:06:39.326 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.MinimumSetOrderConstraint 2024-09-21T10:06:39.329 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.MinimumSetOrderConstraint (3 ms) 2024-09-21T10:06:39.329 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ActiveSet 2024-09-21T10:06:39.332 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ActiveSet (3 ms) 2024-09-21T10:06:39.332 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ActiveSetStale 2024-09-21T10:06:39.335 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ActiveSetStale (3 ms) 2024-09-21T10:06:39.335 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.CreateDuplicate 2024-09-21T10:06:39.338 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.CreateDuplicate (3 ms) 2024-09-21T10:06:39.338 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.GetClient 2024-09-21T10:06:39.341 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.GetClient (3 ms) 2024-09-21T10:06:39.341 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientRegister 2024-09-21T10:06:39.343 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientRegister (2 ms) 2024-09-21T10:06:39.343 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientRegisterDuplicate 2024-09-21T10:06:39.347 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientRegisterDuplicate (4 ms) 2024-09-21T10:06:39.347 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientUpdateData 2024-09-21T10:06:39.351 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientUpdateData (4 ms) 2024-09-21T10:06:39.351 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientUpdateState 2024-09-21T10:06:39.357 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientUpdateState (6 ms) 2024-09-21T10:06:39.357 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientUnregister 2024-09-21T10:06:39.360 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientUnregister (3 ms) 2024-09-21T10:06:39.360 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientUnregisterDNE 2024-09-21T10:06:39.364 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientUnregisterDNE (4 ms) 2024-09-21T10:06:39.364 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientUnregisterPruneTags 2024-09-21T10:06:39.466 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientUnregisterPruneTags (102 ms) 2024-09-21T10:06:39.466 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientCommit 2024-09-21T10:06:39.470 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientCommit (4 ms) 2024-09-21T10:06:39.470 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientCommitInvalid 2024-09-21T10:06:39.473 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientCommitInvalid (3 ms) 2024-09-21T10:06:39.473 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientCommitDNE 2024-09-21T10:06:39.474 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientCommitDNE (1 ms) 2024-09-21T10:06:39.474 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.ClientList 2024-09-21T10:06:39.512 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.ClientList (38 ms) 2024-09-21T10:06:39.512 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.GetNextTagTid 2024-09-21T10:06:39.517 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.GetNextTagTid (5 ms) 2024-09-21T10:06:39.517 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.TagCreate 2024-09-21T10:06:39.526 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.TagCreate (9 ms) 2024-09-21T10:06:39.526 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.TagCreatePrunesTags 2024-09-21T10:06:39.533 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.TagCreatePrunesTags (7 ms) 2024-09-21T10:06:39.533 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.TagList 2024-09-21T10:06:39.621 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.TagList (88 ms) 2024-09-21T10:06:39.697 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.GuardAppend 2024-09-21T10:06:39.698 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.GuardAppend (3 ms) 2024-09-21T10:06:39.698 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.GuardAppendDNE 2024-09-21T10:06:39.698 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.GuardAppendDNE (0 ms) 2024-09-21T10:06:39.698 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.GuardAppendOverflow 2024-09-21T10:06:39.699 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.GuardAppendOverflow (2 ms) 2024-09-21T10:06:39.699 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsJournal.Append 2024-09-21T10:06:39.699 INFO:tasks.workunit.client.0.smithi033.stdout:padding is not enabled 2024-09-21T10:06:39.699 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsJournal.Append (12 ms) 2024-09-21T10:06:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:39 smithi033 ceph-mon[98546]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T10:06:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:39 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2524956369' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:39 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:39.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:39 smithi033 ceph-mon[98546]: pgmap v1029: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:39.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:39 smithi033 ceph-mon[96568]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T10:06:39.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:39 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2524956369' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:39.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:39 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:39.748 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:39 smithi033 ceph-mon[96568]: pgmap v1029: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:39 smithi105 ceph-mon[86649]: osdmap e914: 8 total, 8 up, 8 in 2024-09-21T10:06:39.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:39 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2524956369' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:39.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:39 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:39.781 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:39 smithi105 ceph-mon[86649]: pgmap v1029: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:40.308 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 27 tests from TestClsJournal (340 ms total) 2024-09-21T10:06:40.308 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:06:40.308 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:06:40.308 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 27 tests from 1 test suite ran. (2876 ms total) 2024-09-21T10:06:40.308 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 27 tests. 2024-09-21T10:06:40.310 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T10:06:40.310 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:06:40.380 INFO:tasks.workunit:Running workunit cls/test_cls_lock.sh... 2024-09-21T10:06:40.380 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_lock.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_lock.sh 2024-09-21T10:06:40.451 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:06:40.451 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 11 tests from 1 test suite. 2024-09-21T10:06:40.451 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:06:40.451 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 11 tests from ClsLock 2024-09-21T10:06:40.452 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestMultiLocking 2024-09-21T10:06:40.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:40 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:40.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:40 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:40.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:40 smithi105 ceph-mon[86649]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T10:06:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:06:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:06:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:40 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:40 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:40.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:40 smithi033 ceph-mon[98546]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T10:06:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:40 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:40.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:40 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129419-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:40.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:40 smithi033 ceph-mon[96568]: osdmap e915: 8 total, 8 up, 8 in 2024-09-21T10:06:41.620 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:41 smithi033 ceph-mon[96568]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T10:06:41.620 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:41 smithi033 ceph-mon[96568]: pgmap v1032: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:41.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:41 smithi033 ceph-mon[98546]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T10:06:41.620 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:41 smithi033 ceph-mon[98546]: pgmap v1032: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:41 smithi105 ceph-mon[86649]: osdmap e916: 8 total, 8 up, 8 in 2024-09-21T10:06:41.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:41 smithi105 ceph-mon[86649]: pgmap v1032: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:06:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:06:41] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-21T10:06:42.349 INFO:tasks.workunit.client.0.smithi033.stdout:lock: mylock 2024-09-21T10:06:42.350 INFO:tasks.workunit.client.0.smithi033.stdout: lock_type: exclusive 2024-09-21T10:06:42.350 INFO:tasks.workunit.client.0.smithi033.stdout: tag: 2024-09-21T10:06:42.350 INFO:tasks.workunit.client.0.smithi033.stdout: lockers: 2024-09-21T10:06:42.350 INFO:tasks.workunit.client.0.smithi033.stdout: client.44735 expiration=2024-09-21T10:08:42.344647+0000 addr=v1:172.21.15.33:0/3557007996 cookie= 2024-09-21T10:06:42.354 INFO:tasks.workunit.client.0.smithi033.stdout:lock: mylock 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: lock_type: shared 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: tag: 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: lockers: 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: client.44735 expiration=2024-09-21T10:08:42.351404+0000 addr=v1:172.21.15.33:0/3557007996 cookie= 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: client.44741 expiration=0.000000 addr=v1:172.21.15.33:0/142361939 cookie= 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout:lock: mylock 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: lock_type: shared 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: tag: 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: lockers: 2024-09-21T10:06:42.355 INFO:tasks.workunit.client.0.smithi033.stdout: client.44741 expiration=0.000000 addr=v1:172.21.15.33:0/142361939 cookie= 2024-09-21T10:06:42.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:42.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:42.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[96568]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T10:06:42.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3557007996' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:42.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:42.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:42.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:42.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[98546]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T10:06:42.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3557007996' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:42.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:42 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:42 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:42 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:42 smithi105 ceph-mon[86649]: osdmap e917: 8 total, 8 up, 8 in 2024-09-21T10:06:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:42 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3557007996' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:42.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:42 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:43.335 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestMultiLocking (2884 ms) 2024-09-21T10:06:43.335 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestMeta 2024-09-21T10:06:43.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:43 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:43.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:43 smithi033 ceph-mon[98546]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T10:06:43.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:43 smithi033 ceph-mon[98546]: pgmap v1035: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:43.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:43 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:43.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:43 smithi033 ceph-mon[96568]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T10:06:43.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:43 smithi033 ceph-mon[96568]: pgmap v1035: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:43 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:43 smithi105 ceph-mon[86649]: osdmap e918: 8 total, 8 up, 8 in 2024-09-21T10:06:43.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:43 smithi105 ceph-mon[86649]: pgmap v1035: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:44.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:44 smithi033 ceph-mon[98546]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T10:06:44.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:44 smithi033 ceph-mon[96568]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T10:06:44.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:44 smithi105 ceph-mon[86649]: osdmap e919: 8 total, 8 up, 8 in 2024-09-21T10:06:45.379 INFO:tasks.workunit.client.0.smithi033.stdout:lock: mylock 2024-09-21T10:06:45.379 INFO:tasks.workunit.client.0.smithi033.stdout: lock_type: shared 2024-09-21T10:06:45.379 INFO:tasks.workunit.client.0.smithi033.stdout: tag: 2024-09-21T10:06:45.379 INFO:tasks.workunit.client.0.smithi033.stdout: lockers: 2024-09-21T10:06:45.379 INFO:tasks.workunit.client.0.smithi033.stdout: client.34839 expiration=0.000000 addr=v1:172.21.15.33:0/265271391 cookie= 2024-09-21T10:06:45.381 INFO:tasks.workunit.client.0.smithi033.stdout:lock: mylock 2024-09-21T10:06:45.381 INFO:tasks.workunit.client.0.smithi033.stdout: lock_type: exclusive 2024-09-21T10:06:45.381 INFO:tasks.workunit.client.0.smithi033.stdout: tag: new_tag 2024-09-21T10:06:45.381 INFO:tasks.workunit.client.0.smithi033.stdout: lockers: 2024-09-21T10:06:45.382 INFO:tasks.workunit.client.0.smithi033.stdout: client.44747 expiration=0.000000 addr=v1:172.21.15.33:0/1033068077 cookie= 2024-09-21T10:06:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:45 smithi033 ceph-mon[96568]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T10:06:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:45 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1033068077' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:45 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:45.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:45 smithi033 ceph-mon[96568]: pgmap v1038: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:45.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:45 smithi033 ceph-mon[98546]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T10:06:45.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:45 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1033068077' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:45.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:45 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:45.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:45 smithi033 ceph-mon[98546]: pgmap v1038: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:45 smithi105 ceph-mon[86649]: osdmap e920: 8 total, 8 up, 8 in 2024-09-21T10:06:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:45 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1033068077' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:45 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:45.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:45 smithi105 ceph-mon[86649]: pgmap v1038: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:46.355 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestMeta (3019 ms) 2024-09-21T10:06:46.355 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestCookie 2024-09-21T10:06:46.648 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:46.648 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:46.648 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[96568]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T10:06:46.648 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:06:46.648 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:46.648 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:46.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:46.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[98546]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T10:06:46.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:06:46.649 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:46 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:46 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:46 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:46 smithi105 ceph-mon[86649]: osdmap e921: 8 total, 8 up, 8 in 2024-09-21T10:06:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:06:46.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:46 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:06:47.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:47 smithi033 ceph-mon[98546]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T10:06:47.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:47 smithi033 ceph-mon[98546]: pgmap v1041: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:47.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:47 smithi033 ceph-mon[96568]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T10:06:47.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:47 smithi033 ceph-mon[96568]: pgmap v1041: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:47 smithi105 ceph-mon[86649]: osdmap e922: 8 total, 8 up, 8 in 2024-09-21T10:06:47.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:47 smithi105 ceph-mon[86649]: pgmap v1041: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:48.389 INFO:tasks.workunit.client.0.smithi033.stdout:lock: mylock 2024-09-21T10:06:48.389 INFO:tasks.workunit.client.0.smithi033.stdout: lock_type: exclusive 2024-09-21T10:06:48.389 INFO:tasks.workunit.client.0.smithi033.stdout: tag: 2024-09-21T10:06:48.389 INFO:tasks.workunit.client.0.smithi033.stdout: lockers: 2024-09-21T10:06:48.391 INFO:tasks.workunit.client.0.smithi033.stdout:lock: mylock 2024-09-21T10:06:48.391 INFO:tasks.workunit.client.0.smithi033.stdout: lock_type: shared 2024-09-21T10:06:48.391 INFO:tasks.workunit.client.0.smithi033.stdout: tag: 2024-09-21T10:06:48.391 INFO:tasks.workunit.client.0.smithi033.stdout: lockers: 2024-09-21T10:06:48.391 INFO:tasks.workunit.client.0.smithi033.stdout: client.34845 expiration=0.000000 addr=v1:172.21.15.33:0/3589071887 cookie= 2024-09-21T10:06:48.391 INFO:tasks.workunit.client.0.smithi033.stdout: client.34845 expiration=0.000000 addr=v1:172.21.15.33:0/3589071887 cookie=new cookie 2024-09-21T10:06:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:48 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:48 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:48 smithi033 ceph-mon[98546]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T10:06:48.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:48 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3589071887' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:48.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:48 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:48.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:48 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:48.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:48 smithi033 ceph-mon[96568]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T10:06:48.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:48 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3589071887' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:48 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:48 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:48 smithi105 ceph-mon[86649]: osdmap e923: 8 total, 8 up, 8 in 2024-09-21T10:06:48.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:48 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3589071887' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:49.382 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestCookie (3026 ms) 2024-09-21T10:06:49.382 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestMultipleLocks 2024-09-21T10:06:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:49 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3589071887' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:49 smithi033 ceph-mon[98546]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T10:06:49.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:49 smithi033 ceph-mon[98546]: pgmap v1044: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:49.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:49 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3589071887' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:49.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:49 smithi033 ceph-mon[96568]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T10:06:49.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:49 smithi033 ceph-mon[96568]: pgmap v1044: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:49 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3589071887' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:49 smithi105 ceph-mon[86649]: osdmap e924: 8 total, 8 up, 8 in 2024-09-21T10:06:49.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:49 smithi105 ceph-mon[86649]: pgmap v1044: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:50.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:50 smithi105 ceph-mon[86649]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T10:06:50.531 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:06:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:06:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:06:50.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:50 smithi033 ceph-mon[98546]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T10:06:50.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:50 smithi033 ceph-mon[96568]: osdmap e925: 8 total, 8 up, 8 in 2024-09-21T10:06:51.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:51 smithi033 ceph-mon[96568]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T10:06:51.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:51 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3526152651' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:51.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:51 smithi033 ceph-mon[96568]: pgmap v1047: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:51.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:51 smithi033 ceph-mon[98546]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T10:06:51.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:51 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3526152651' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:51.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:51 smithi033 ceph-mon[98546]: pgmap v1047: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:51.747 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:06:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:06:51] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-21T10:06:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:51 smithi105 ceph-mon[86649]: osdmap e926: 8 total, 8 up, 8 in 2024-09-21T10:06:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:51 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3526152651' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:51.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:51 smithi105 ceph-mon[86649]: pgmap v1047: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:52.396 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestMultipleLocks (3013 ms) 2024-09-21T10:06:52.396 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestLockDuration 2024-09-21T10:06:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:52 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:52 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3526152651' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:52.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:52 smithi033 ceph-mon[98546]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T10:06:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:52 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:52 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3526152651' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:52.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:52 smithi033 ceph-mon[96568]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T10:06:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:52 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:06:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:52 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3526152651' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:52.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:52 smithi105 ceph-mon[86649]: osdmap e927: 8 total, 8 up, 8 in 2024-09-21T10:06:53.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[98546]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T10:06:53.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[98546]: pgmap v1050: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:53.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:53.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:53.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:53.747 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:06:53.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[96568]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T10:06:53.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[96568]: pgmap v1050: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:53.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:53.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:53.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:53.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:06:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:53 smithi105 ceph-mon[86649]: osdmap e928: 8 total, 8 up, 8 in 2024-09-21T10:06:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:53 smithi105 ceph-mon[86649]: pgmap v1050: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:06:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:06:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:06:53.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:06:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:54 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:54 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:06:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:54 smithi033 ceph-mon[98546]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T10:06:54.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:54 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1982229830' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:54.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:54 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:54.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:54 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:06:54.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:54 smithi033 ceph-mon[96568]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T10:06:54.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:54 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1982229830' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:54 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:06:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:54 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:06:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:54 smithi105 ceph-mon[86649]: osdmap e929: 8 total, 8 up, 8 in 2024-09-21T10:06:54.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:54 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1982229830' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-5","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:06:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:55 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1982229830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:55 smithi033 ceph-mon[98546]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T10:06:55.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:55 smithi033 ceph-mon[98546]: pgmap v1053: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:55.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:55 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1982229830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:55.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:55 smithi033 ceph-mon[96568]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T10:06:55.747 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:55 smithi033 ceph-mon[96568]: pgmap v1053: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:55.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:55 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1982229830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:06:55.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:55 smithi105 ceph-mon[86649]: osdmap e930: 8 total, 8 up, 8 in 2024-09-21T10:06:55.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:55 smithi105 ceph-mon[86649]: pgmap v1053: 137 pgs: 15 creating+peering, 17 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:56.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:56 smithi033 ceph-mon[98546]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T10:06:56.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:56 smithi033 ceph-mon[96568]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T10:06:56.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:56 smithi105 ceph-mon[86649]: osdmap e931: 8 total, 8 up, 8 in 2024-09-21T10:06:57.746 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:57 smithi033 ceph-mon[98546]: pgmap v1055: 137 pgs: 15 creating+peering, 8 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:57.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:57 smithi033 ceph-mon[96568]: pgmap v1055: 137 pgs: 15 creating+peering, 8 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:57.780 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:57 smithi105 ceph-mon[86649]: pgmap v1055: 137 pgs: 15 creating+peering, 8 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:06:59.559 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestLockDuration (7164 ms) 2024-09-21T10:06:59.559 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestAssertLocked 2024-09-21T10:06:59.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:06:59 smithi033 ceph-mon[98546]: pgmap v1056: 137 pgs: 15 creating+peering, 122 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:06:59.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:06:59 smithi033 ceph-mon[96568]: pgmap v1056: 137 pgs: 15 creating+peering, 122 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:00.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:06:59 smithi105 ceph-mon[86649]: pgmap v1056: 137 pgs: 15 creating+peering, 122 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:00.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:07:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:07:00.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:00.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:00 smithi033 ceph-mon[98546]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T10:07:00.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:00 smithi033 ceph-mon[96568]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T10:07:01.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:00 smithi105 ceph-mon[86649]: osdmap e932: 8 total, 8 up, 8 in 2024-09-21T10:07:01.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:07:01] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-21T10:07:01.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[98546]: pgmap v1058: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[98546]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3493183333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[96568]: pgmap v1058: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[96568]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3493183333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:07:01.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:01 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:01 smithi105 ceph-mon[86649]: pgmap v1058: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:01 smithi105 ceph-mon[86649]: osdmap e933: 8 total, 8 up, 8 in 2024-09-21T10:07:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:01 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3493183333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:01 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:07:02.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:01 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:02.594 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestAssertLocked (3034 ms) 2024-09-21T10:07:02.594 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestSetCookie 2024-09-21T10:07:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:02 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:02.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:02 smithi033 ceph-mon[98546]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T10:07:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:02 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:02.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:02 smithi033 ceph-mon[96568]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T10:07:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:02 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:03.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:02 smithi105 ceph-mon[86649]: osdmap e934: 8 total, 8 up, 8 in 2024-09-21T10:07:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:03 smithi033 ceph-mon[98546]: pgmap v1061: 137 pgs: 1 creating+activating, 7 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:03.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:03 smithi033 ceph-mon[98546]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T10:07:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:03 smithi033 ceph-mon[96568]: pgmap v1061: 137 pgs: 1 creating+activating, 7 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:03.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:03 smithi033 ceph-mon[96568]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T10:07:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:03 smithi105 ceph-mon[86649]: pgmap v1061: 137 pgs: 1 creating+activating, 7 creating+peering, 24 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:04.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:03 smithi105 ceph-mon[86649]: osdmap e935: 8 total, 8 up, 8 in 2024-09-21T10:07:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:04 smithi033 ceph-mon[98546]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T10:07:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:04 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2769355333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:04.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:04 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:04.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:04 smithi033 ceph-mon[96568]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T10:07:04.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:04 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2769355333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:04.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:04 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:05.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:04 smithi105 ceph-mon[86649]: osdmap e936: 8 total, 8 up, 8 in 2024-09-21T10:07:05.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:04 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2769355333' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:05.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:04 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:05.613 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestSetCookie (3018 ms) 2024-09-21T10:07:05.613 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestRenew 2024-09-21T10:07:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:05 smithi033 ceph-mon[98546]: pgmap v1064: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:05 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:05 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:05.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:05 smithi033 ceph-mon[98546]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T10:07:05.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:05 smithi033 ceph-mon[96568]: pgmap v1064: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:05.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:05 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:05.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:05 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:05.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:05 smithi033 ceph-mon[96568]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T10:07:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:05 smithi105 ceph-mon[86649]: pgmap v1064: 137 pgs: 10 creating+peering, 22 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:05 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:05 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:06.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:05 smithi105 ceph-mon[86649]: osdmap e937: 8 total, 8 up, 8 in 2024-09-21T10:07:06.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:06 smithi033 ceph-mon[98546]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T10:07:06.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:06 smithi033 ceph-mon[96568]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T10:07:07.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:06 smithi105 ceph-mon[86649]: osdmap e938: 8 total, 8 up, 8 in 2024-09-21T10:07:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[96568]: pgmap v1067: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:07:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[96568]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T10:07:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/550114401' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:07.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:07.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[98546]: pgmap v1067: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:07.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:07.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:07:07.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[98546]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T10:07:07.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/550114401' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:07.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:07 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:07 smithi105 ceph-mon[86649]: pgmap v1067: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:07 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:07 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:07:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:07 smithi105 ceph-mon[86649]: osdmap e939: 8 total, 8 up, 8 in 2024-09-21T10:07:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:07 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/550114401' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:08.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:07 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:08 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:08.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:08 smithi033 ceph-mon[98546]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T10:07:08.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:08 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:08.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:08 smithi033 ceph-mon[96568]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T10:07:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:08 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:09.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:08 smithi105 ceph-mon[86649]: osdmap e940: 8 total, 8 up, 8 in 2024-09-21T10:07:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:09 smithi033 ceph-mon[98546]: pgmap v1070: 137 pgs: 27 unknown, 110 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:09.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:09 smithi033 ceph-mon[98546]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T10:07:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:09 smithi033 ceph-mon[96568]: pgmap v1070: 137 pgs: 27 unknown, 110 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:09.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:09 smithi033 ceph-mon[96568]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T10:07:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:09 smithi105 ceph-mon[86649]: pgmap v1070: 137 pgs: 27 unknown, 110 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:10.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:09 smithi105 ceph-mon[86649]: osdmap e941: 8 total, 8 up, 8 in 2024-09-21T10:07:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:07:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:07:10.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:11.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:11 smithi033 ceph-mon[98546]: pgmap v1072: 137 pgs: 10 unknown, 127 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:11.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:11 smithi033 ceph-mon[96568]: pgmap v1072: 137 pgs: 10 unknown, 127 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:11.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:07:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:07:11] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.51.0" 2024-09-21T10:07:12.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:11 smithi105 ceph-mon[86649]: pgmap v1072: 137 pgs: 10 unknown, 127 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:13.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:13 smithi033 ceph-mon[98546]: pgmap v1073: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:13.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:13 smithi033 ceph-mon[96568]: pgmap v1073: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:14.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:13 smithi105 ceph-mon[86649]: pgmap v1073: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:15.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:15 smithi033 ceph-mon[98546]: pgmap v1074: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 129 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:15.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:15 smithi033 ceph-mon[96568]: pgmap v1074: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 129 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:16.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:15 smithi105 ceph-mon[86649]: pgmap v1074: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 129 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:16.897 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:16 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:16.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:16 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:17.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:16 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:17.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:17 smithi033 ceph-mon[98546]: pgmap v1075: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 115 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:17.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:17 smithi033 ceph-mon[96568]: pgmap v1075: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 115 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:18.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:17 smithi105 ceph-mon[86649]: pgmap v1075: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 115 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:19.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:19 smithi033 ceph-mon[98546]: pgmap v1076: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:19.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:19 smithi033 ceph-mon[96568]: pgmap v1076: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:20.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:19 smithi105 ceph-mon[86649]: pgmap v1076: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 102 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:20.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:07:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:07:20.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:21.998 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:07:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:07:21] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.51.0" 2024-09-21T10:07:21.998 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:21 smithi033 ceph-mon[98546]: pgmap v1077: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 86 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:21.999 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:21 smithi033 ceph-mon[96568]: pgmap v1077: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 86 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:22.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:21 smithi105 ceph-mon[86649]: pgmap v1077: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 86 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:23.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:23 smithi033 ceph-mon[98546]: pgmap v1078: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T10:07:23.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:23 smithi033 ceph-mon[96568]: pgmap v1078: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T10:07:24.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:23 smithi105 ceph-mon[86649]: pgmap v1078: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 341 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T10:07:25.673 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestRenew (20060 ms) 2024-09-21T10:07:25.674 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralBasic 2024-09-21T10:07:25.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:25 smithi033 ceph-mon[98546]: pgmap v1079: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:25.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:25 smithi033 ceph-mon[96568]: pgmap v1079: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:26.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:25 smithi105 ceph-mon[86649]: pgmap v1079: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 255 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:26.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:26 smithi033 ceph-mon[96568]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T10:07:26.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:26 smithi033 ceph-mon[98546]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T10:07:27.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:26 smithi105 ceph-mon[86649]: osdmap e942: 8 total, 8 up, 8 in 2024-09-21T10:07:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:27 smithi033 ceph-mon[96568]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:27 smithi033 ceph-mon[96568]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T10:07:27.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:27 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2827311901' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:27.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:27 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:27 smithi033 ceph-mon[98546]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:27 smithi033 ceph-mon[98546]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T10:07:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:27 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2827311901' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:27.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:27 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:27 smithi105 ceph-mon[86649]: pgmap v1081: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:27 smithi105 ceph-mon[86649]: osdmap e943: 8 total, 8 up, 8 in 2024-09-21T10:07:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:27 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2827311901' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:28.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:27 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:28 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:28.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:28 smithi033 ceph-mon[96568]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T10:07:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:28 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:28.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:28 smithi033 ceph-mon[98546]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T10:07:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:28 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:29.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:28 smithi105 ceph-mon[86649]: osdmap e944: 8 total, 8 up, 8 in 2024-09-21T10:07:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:29 smithi033 ceph-mon[96568]: pgmap v1084: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:29.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:29 smithi033 ceph-mon[96568]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T10:07:29.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:29 smithi033 ceph-mon[98546]: pgmap v1084: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:29.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:29 smithi033 ceph-mon[98546]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T10:07:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:29 smithi105 ceph-mon[86649]: pgmap v1084: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:30.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:29 smithi105 ceph-mon[86649]: osdmap e945: 8 total, 8 up, 8 in 2024-09-21T10:07:30.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:07:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:07:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:31.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:07:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:07:31] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.51.0" 2024-09-21T10:07:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:31 smithi033 ceph-mon[98546]: pgmap v1086: 137 pgs: 3 creating+peering, 9 unknown, 125 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:31.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:31 smithi033 ceph-mon[96568]: pgmap v1086: 137 pgs: 3 creating+peering, 9 unknown, 125 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:31.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:31 smithi105 ceph-mon[86649]: pgmap v1086: 137 pgs: 3 creating+peering, 9 unknown, 125 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:32.718 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestExclusiveEphemeralBasic (7045 ms) 2024-09-21T10:07:32.719 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealEphemeral 2024-09-21T10:07:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:33 smithi033 ceph-mon[98546]: pgmap v1087: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T10:07:33.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:33 smithi033 ceph-mon[98546]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T10:07:33.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:33 smithi033 ceph-mon[96568]: pgmap v1087: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T10:07:33.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:33 smithi033 ceph-mon[96568]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T10:07:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:33 smithi105 ceph-mon[86649]: pgmap v1087: 137 pgs: 3 creating+peering, 134 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 170 B/s rd, 0 B/s wr, 1 op/s 2024-09-21T10:07:34.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:33 smithi105 ceph-mon[86649]: osdmap e946: 8 total, 8 up, 8 in 2024-09-21T10:07:34.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:34 smithi033 ceph-mon[96568]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T10:07:34.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:34 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3579184339' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:34.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:34 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:34 smithi033 ceph-mon[98546]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T10:07:34.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:34 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3579184339' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:34.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:34 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:34 smithi105 ceph-mon[86649]: osdmap e947: 8 total, 8 up, 8 in 2024-09-21T10:07:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:34 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3579184339' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:35.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:34 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:35 smithi033 ceph-mon[98546]: pgmap v1090: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:35.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:35 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:35 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:35.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:35 smithi033 ceph-mon[98546]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T10:07:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:35 smithi033 ceph-mon[96568]: pgmap v1090: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:35 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:35 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:35.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:35 smithi033 ceph-mon[96568]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T10:07:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:35 smithi105 ceph-mon[86649]: pgmap v1090: 137 pgs: 5 creating+peering, 27 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:35 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:36.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:35 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:36.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:35 smithi105 ceph-mon[86649]: osdmap e948: 8 total, 8 up, 8 in 2024-09-21T10:07:36.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:36 smithi033 ceph-mon[96568]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T10:07:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:36 smithi033 ceph-mon[98546]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T10:07:37.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:36 smithi105 ceph-mon[86649]: osdmap e949: 8 total, 8 up, 8 in 2024-09-21T10:07:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:37 smithi033 ceph-mon[96568]: pgmap v1093: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:37 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:37 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:07:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:37 smithi033 ceph-mon[98546]: pgmap v1093: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:37.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:37 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:37.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:37 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:07:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:37 smithi105 ceph-mon[86649]: pgmap v1093: 137 pgs: 6 creating+peering, 18 unknown, 113 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:37 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:38.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:37 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:07:39.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:39 smithi033 ceph-mon[98546]: pgmap v1094: 137 pgs: 6 creating+peering, 11 unknown, 120 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:39 smithi033 ceph-mon[96568]: pgmap v1094: 137 pgs: 6 creating+peering, 11 unknown, 120 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:40.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:39 smithi105 ceph-mon[86649]: pgmap v1094: 137 pgs: 6 creating+peering, 11 unknown, 120 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:07:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:07:40.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:40.757 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealEphemeral (8039 ms) 2024-09-21T10:07:40.758 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsLock.TestExclusiveEphemeralStealExclusive 2024-09-21T10:07:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:41 smithi033 ceph-mon[96568]: pgmap v1095: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:41 smithi033 ceph-mon[96568]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T10:07:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:41 smithi033 ceph-mon[98546]: pgmap v1095: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:41.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:41 smithi033 ceph-mon[98546]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T10:07:41.997 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:07:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:07:41] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.51.0" 2024-09-21T10:07:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:41 smithi105 ceph-mon[86649]: pgmap v1095: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:42.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:41 smithi105 ceph-mon[86649]: osdmap e950: 8 total, 8 up, 8 in 2024-09-21T10:07:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:42 smithi105 ceph-mon[86649]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T10:07:43.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:42 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2261234979' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:43.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:42 smithi033 ceph-mon[96568]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T10:07:43.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:42 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2261234979' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:43.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:42 smithi033 ceph-mon[98546]: osdmap e951: 8 total, 8 up, 8 in 2024-09-21T10:07:43.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:42 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2261234979' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:43 smithi105 ceph-mon[86649]: pgmap v1098: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:44.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:43 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:43 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2261234979' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:44.031 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:43 smithi105 ceph-mon[86649]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T10:07:44.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:43 smithi033 ceph-mon[96568]: pgmap v1098: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:44.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:43 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:44.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:43 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2261234979' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:44.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:43 smithi033 ceph-mon[96568]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T10:07:44.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:43 smithi033 ceph-mon[98546]: pgmap v1098: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:44.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:43 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:44.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:43 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2261234979' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129474-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:44.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:43 smithi033 ceph-mon[98546]: osdmap e952: 8 total, 8 up, 8 in 2024-09-21T10:07:45.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:44 smithi105 ceph-mon[86649]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T10:07:45.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:44 smithi033 ceph-mon[98546]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T10:07:45.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:44 smithi033 ceph-mon[96568]: osdmap e953: 8 total, 8 up, 8 in 2024-09-21T10:07:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:45 smithi105 ceph-mon[86649]: pgmap v1101: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:45 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:45 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:07:46.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:45 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:46.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:45 smithi033 ceph-mon[98546]: pgmap v1101: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:46.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:45 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:46.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:45 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:07:46.148 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:45 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:46.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:45 smithi033 ceph-mon[96568]: pgmap v1101: 137 pgs: 7 creating+peering, 16 unknown, 114 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:46.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:45 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:46.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:45 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:07:46.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:45 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:07:48.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:47 smithi033 ceph-mon[98546]: pgmap v1102: 137 pgs: 7 creating+peering, 4 unknown, 126 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:48.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:47 smithi033 ceph-mon[96568]: pgmap v1102: 137 pgs: 7 creating+peering, 4 unknown, 126 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:48.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:47 smithi105 ceph-mon[86649]: pgmap v1102: 137 pgs: 7 creating+peering, 4 unknown, 126 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:48.800 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsLock.TestExclusiveEphemeralStealExclusive (8042 ms) 2024-09-21T10:07:48.800 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 11 tests from ClsLock (68348 ms total) 2024-09-21T10:07:48.800 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:07:48.800 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:07:48.800 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 11 tests from 1 test suite ran. (68348 ms total) 2024-09-21T10:07:48.800 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 11 tests. 2024-09-21T10:07:48.803 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T10:07:48.803 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:07:48.873 INFO:tasks.workunit:Running workunit cls/test_cls_log.sh... 2024-09-21T10:07:48.874 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_log.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_log.sh 2024-09-21T10:07:48.944 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:07:48.944 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-21T10:07:48.944 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:07:48.944 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 4 tests from cls_log 2024-09-21T10:07:48.944 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_log.test_log_add_same_time 2024-09-21T10:07:50.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:49 smithi105 ceph-mon[86649]: pgmap v1103: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:50.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:49 smithi105 ceph-mon[86649]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T10:07:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:49 smithi033 ceph-mon[96568]: pgmap v1103: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:50.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:49 smithi033 ceph-mon[96568]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T10:07:50.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:49 smithi033 ceph-mon[98546]: pgmap v1103: 137 pgs: 137 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s rd, 0 B/s wr, 0 op/s 2024-09-21T10:07:50.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:49 smithi033 ceph-mon[98546]: osdmap e954: 8 total, 8 up, 8 in 2024-09-21T10:07:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:07:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:07:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:07:51.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:50 smithi033 ceph-mon[98546]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T10:07:51.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:50 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1075889379' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:51.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:50 smithi033 ceph-mon[96568]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T10:07:51.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:50 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1075889379' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:51.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:50 smithi105 ceph-mon[86649]: osdmap e955: 8 total, 8 up, 8 in 2024-09-21T10:07:51.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:50 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1075889379' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:51.817 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_log.test_log_add_same_time (2873 ms) 2024-09-21T10:07:51.817 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_log.test_log_add_different_time 2024-09-21T10:07:51.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:07:51] "GET /metrics HTTP/1.1" 200 36457 "" "Prometheus/2.51.0" 2024-09-21T10:07:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-mon[96568]: pgmap v1106: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:51.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1075889379' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:51.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-mon[96568]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T10:07:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-mon[98546]: pgmap v1106: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1075889379' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:51.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:51 smithi033 ceph-mon[98546]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T10:07:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:51 smithi105 ceph-mon[86649]: pgmap v1106: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:51 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:51 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1075889379' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:52.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:51 smithi105 ceph-mon[86649]: osdmap e956: 8 total, 8 up, 8 in 2024-09-21T10:07:53.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:52 smithi033 ceph-mon[98546]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T10:07:53.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:52 smithi033 ceph-mon[96568]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T10:07:53.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:52 smithi105 ceph-mon[86649]: osdmap e957: 8 total, 8 up, 8 in 2024-09-21T10:07:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:54.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2487655782' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T10:07:54.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2487655782' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:54.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:54.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:54.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:54.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:53 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:07:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: pgmap v1109: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:07:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:07:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: osdmap e958: 8 total, 8 up, 8 in 2024-09-21T10:07:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2487655782' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:54.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-21T10:07:54.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-21T10:07:54.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-21T10:07:54.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:53 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:07:54.842 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_log.test_log_add_different_time (3025 ms) 2024-09-21T10:07:54.843 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_log.trim_by_time 2024-09-21T10:07:55.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:54 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2487655782' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:55.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:54 smithi033 ceph-mon[98546]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T10:07:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:54 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2487655782' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:55.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:54 smithi033 ceph-mon[96568]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T10:07:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:54 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2487655782' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:55.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:54 smithi105 ceph-mon[86649]: osdmap e959: 8 total, 8 up, 8 in 2024-09-21T10:07:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:55 smithi033 ceph-mon[98546]: pgmap v1112: 137 pgs: 7 creating+peering, 24 unknown, 106 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:56.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:55 smithi033 ceph-mon[98546]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T10:07:56.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:55 smithi033 ceph-mon[96568]: pgmap v1112: 137 pgs: 7 creating+peering, 24 unknown, 106 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:56.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:55 smithi033 ceph-mon[96568]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T10:07:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:55 smithi105 ceph-mon[86649]: pgmap v1112: 137 pgs: 7 creating+peering, 24 unknown, 106 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:56.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:55 smithi105 ceph-mon[86649]: osdmap e960: 8 total, 8 up, 8 in 2024-09-21T10:07:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:56 smithi033 ceph-mon[98546]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T10:07:57.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:56 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/502965907' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:57.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:56 smithi033 ceph-mon[96568]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T10:07:57.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:56 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/502965907' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:56 smithi105 ceph-mon[86649]: osdmap e961: 8 total, 8 up, 8 in 2024-09-21T10:07:57.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:56 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/502965907' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:07:57.864 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_log.trim_by_time (3022 ms) 2024-09-21T10:07:57.864 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] cls_log.trim_by_marker 2024-09-21T10:07:58.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:57 smithi033 ceph-mon[98546]: pgmap v1115: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:57 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:57 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/502965907' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:58.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:57 smithi033 ceph-mon[98546]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T10:07:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:57 smithi033 ceph-mon[96568]: pgmap v1115: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:58.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:57 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:57 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/502965907' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:58.248 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:57 smithi033 ceph-mon[96568]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T10:07:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:57 smithi105 ceph-mon[86649]: pgmap v1115: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:07:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:57 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:07:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:57 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/502965907' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:07:58.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:57 smithi105 ceph-mon[86649]: osdmap e962: 8 total, 8 up, 8 in 2024-09-21T10:07:59.182 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:58 smithi105 ceph-mon[86649]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T10:07:59.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:58 smithi033 ceph-mon[98546]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T10:07:59.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:58 smithi033 ceph-mon[96568]: osdmap e963: 8 total, 8 up, 8 in 2024-09-21T10:08:00.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:59 smithi105 ceph-mon[86649]: pgmap v1118: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:00.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:59 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:00.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:59 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:08:00.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:59 smithi105 ceph-mon[86649]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T10:08:00.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:59 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3319722398' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:00.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:07:59 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[98546]: pgmap v1118: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:08:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[98546]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T10:08:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3319722398' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:00.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:00.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[96568]: pgmap v1118: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:00.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:08:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[96568]: osdmap e964: 8 total, 8 up, 8 in 2024-09-21T10:08:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3319722398' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:00.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:07:59 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:00.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:08:00 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:08:00.212Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:00.891 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] cls_log.trim_by_marker (3027 ms) 2024-09-21T10:08:00.891 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 4 tests from cls_log (11948 ms total) 2024-09-21T10:08:00.891 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:08:00.891 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:08:00.892 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 4 tests from 1 test suite ran. (11948 ms total) 2024-09-21T10:08:00.892 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 4 tests. 2024-09-21T10:08:00.893 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T10:08:00.894 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:08:00.964 INFO:tasks.workunit:Running workunit cls/test_cls_numops.sh... 2024-09-21T10:08:00.964 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_numops.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_numops.sh 2024-09-21T10:08:01.036 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:08:01.036 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 4 tests from 1 test suite. 2024-09-21T10:08:01.037 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:08:01.037 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 4 tests from ClsNumOps 2024-09-21T10:08:01.037 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsNumOps.Add 2024-09-21T10:08:01.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:00 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:01.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:00 smithi033 ceph-mon[98546]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T10:08:01.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:00 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:01.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:00 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:01.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:00 smithi033 ceph-mon[96568]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T10:08:01.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:00 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:00 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-129811-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:00 smithi105 ceph-mon[86649]: osdmap e965: 8 total, 8 up, 8 in 2024-09-21T10:08:01.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:00 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:01.881 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:01 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:08:01] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-09-21T10:08:02.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:01 smithi033 ceph-mon[98546]: pgmap v1121: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:02.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:01 smithi033 ceph-mon[98546]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T10:08:02.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:01 smithi033 ceph-mon[96568]: pgmap v1121: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:02.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:01 smithi033 ceph-mon[96568]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T10:08:02.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:01 smithi105 ceph-mon[86649]: pgmap v1121: 137 pgs: 7 creating+peering, 25 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:02.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:01 smithi105 ceph-mon[86649]: osdmap e966: 8 total, 8 up, 8 in 2024-09-21T10:08:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:02 smithi033 ceph-mon[98546]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T10:08:03.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:02 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2624547084' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:03.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:02 smithi033 ceph-mon[96568]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T10:08:03.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:02 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2624547084' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:02 smithi105 ceph-mon[86649]: osdmap e967: 8 total, 8 up, 8 in 2024-09-21T10:08:03.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:02 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2624547084' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:03.908 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsNumOps.Add (2871 ms) 2024-09-21T10:08:03.908 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsNumOps.Sub 2024-09-21T10:08:04.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:03 smithi033 ceph-mon[96568]: pgmap v1124: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:04.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:03 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:04.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:03 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2624547084' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:04.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:03 smithi033 ceph-mon[96568]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T10:08:04.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:03 smithi033 ceph-mon[98546]: pgmap v1124: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:04.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:03 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:04.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:03 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2624547084' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:04.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:03 smithi033 ceph-mon[98546]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T10:08:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:03 smithi105 ceph-mon[86649]: pgmap v1124: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:03 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:03 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2624547084' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:04.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:03 smithi105 ceph-mon[86649]: osdmap e968: 8 total, 8 up, 8 in 2024-09-21T10:08:05.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:04 smithi033 ceph-mon[96568]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T10:08:05.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:04 smithi033 ceph-mon[98546]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T10:08:05.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:04 smithi105 ceph-mon[86649]: osdmap e969: 8 total, 8 up, 8 in 2024-09-21T10:08:06.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[96568]: pgmap v1127: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:06.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:06.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:08:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[96568]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T10:08:06.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3592602474' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:06.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[98546]: pgmap v1127: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:06.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:06.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:08:06.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[98546]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T10:08:06.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:05 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3592602474' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:05 smithi105 ceph-mon[86649]: pgmap v1127: 105 pgs: 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:05 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:05 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:08:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:05 smithi105 ceph-mon[86649]: osdmap e970: 8 total, 8 up, 8 in 2024-09-21T10:08:06.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:05 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3592602474' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-2","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:06.935 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsNumOps.Sub (3028 ms) 2024-09-21T10:08:06.936 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsNumOps.Mul 2024-09-21T10:08:07.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:06 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3592602474' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:07.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:06 smithi033 ceph-mon[96568]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T10:08:07.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:06 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3592602474' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:07.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:06 smithi033 ceph-mon[98546]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T10:08:07.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:06 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3592602474' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:07.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:06 smithi105 ceph-mon[86649]: osdmap e971: 8 total, 8 up, 8 in 2024-09-21T10:08:08.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:07 smithi033 ceph-mon[96568]: pgmap v1130: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:08.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:07 smithi033 ceph-mon[96568]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T10:08:08.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:07 smithi033 ceph-mon[98546]: pgmap v1130: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:08.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:07 smithi033 ceph-mon[98546]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T10:08:08.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:07 smithi105 ceph-mon[86649]: pgmap v1130: 137 pgs: 11 creating+peering, 21 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:08.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:07 smithi105 ceph-mon[86649]: osdmap e972: 8 total, 8 up, 8 in 2024-09-21T10:08:09.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:08 smithi033 ceph-mon[98546]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T10:08:09.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:08 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2626236288' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:09.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:08 smithi033 ceph-mon[96568]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T10:08:09.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:08 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2626236288' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:09.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:08 smithi105 ceph-mon[86649]: osdmap e973: 8 total, 8 up, 8 in 2024-09-21T10:08:09.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:08 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2626236288' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-3","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:09.951 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsNumOps.Mul (3015 ms) 2024-09-21T10:08:09.951 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] ClsNumOps.Div 2024-09-21T10:08:10.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:09 smithi105 ceph-mon[86649]: pgmap v1133: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:10.214 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:09 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:10.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:09 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2626236288' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:10.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:09 smithi105 ceph-mon[86649]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T10:08:10.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:09 smithi033 ceph-mon[98546]: pgmap v1133: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:10.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:09 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:10.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:09 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2626236288' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:10.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:09 smithi033 ceph-mon[98546]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T10:08:10.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:09 smithi033 ceph-mon[96568]: pgmap v1133: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:10.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:09 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:10.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:09 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2626236288' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:10.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:09 smithi033 ceph-mon[96568]: osdmap e974: 8 total, 8 up, 8 in 2024-09-21T10:08:10.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:08:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:08:10.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:11.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:10 smithi033 ceph-mon[98546]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T10:08:11.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:10 smithi033 ceph-mon[96568]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T10:08:11.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:10 smithi105 ceph-mon[86649]: osdmap e975: 8 total, 8 up, 8 in 2024-09-21T10:08:11.962 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:08:11] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.51.0" 2024-09-21T10:08:12.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[98546]: pgmap v1136: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:12.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:12.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:08:12.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[98546]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T10:08:12.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3298948241' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[96568]: pgmap v1136: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:08:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[96568]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T10:08:12.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:11 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3298948241' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:11 smithi105 ceph-mon[86649]: pgmap v1136: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:11 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:11 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:08:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:11 smithi105 ceph-mon[86649]: osdmap e976: 8 total, 8 up, 8 in 2024-09-21T10:08:12.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:11 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3298948241' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-4","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:12.981 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] ClsNumOps.Div (3031 ms) 2024-09-21T10:08:12.982 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 4 tests from ClsNumOps (11945 ms total) 2024-09-21T10:08:12.982 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:08:12.982 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:08:12.982 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 4 tests from 1 test suite ran. (11945 ms total) 2024-09-21T10:08:12.982 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 4 tests. 2024-09-21T10:08:12.986 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-21T10:08:12.986 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:08:13.058 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-21T10:08:13.058 DEBUG:teuthology.orchestra.run.smithi033:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-21T10:08:13.131 INFO:tasks.workunit.client.0.smithi033.stdout:Running main() from gmock_main.cc 2024-09-21T10:08:13.131 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-21T10:08:13.131 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment set-up. 2024-09-21T10:08:13.131 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 70 tests from TestClsRbd 2024-09-21T10:08:13.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:12 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3298948241' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:13.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:12 smithi033 ceph-mon[98546]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T10:08:13.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:12 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3298948241' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:13.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:12 smithi033 ceph-mon[96568]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T10:08:13.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:12 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3298948241' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130001-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:13.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:12 smithi105 ceph-mon[86649]: osdmap e977: 8 total, 8 up, 8 in 2024-09-21T10:08:14.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:13 smithi033 ceph-mon[98546]: pgmap v1139: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:14.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:13 smithi033 ceph-mon[98546]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T10:08:14.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:13 smithi033 ceph-mon[96568]: pgmap v1139: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:14.247 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:13 smithi033 ceph-mon[96568]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T10:08:14.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:13 smithi105 ceph-mon[86649]: pgmap v1139: 137 pgs: 4 creating+peering, 28 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:14.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:13 smithi105 ceph-mon[86649]: osdmap e978: 8 total, 8 up, 8 in 2024-09-21T10:08:14.987 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-21T10:08:15.002 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_all_features (15 ms) 2024-09-21T10:08:15.002 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.copyup 2024-09-21T10:08:15.185 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.copyup (183 ms) 2024-09-21T10:08:15.185 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-21T10:08:15.203 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.sparse_copyup (18 ms) 2024-09-21T10:08:15.204 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-21T10:08:15.211 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2024-09-21T10:08:15.211 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-21T10:08:15.223 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.add_remove_child (12 ms) 2024-09-21T10:08:15.223 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-21T10:08:15.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:14 smithi033 ceph-mon[98546]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T10:08:15.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:14 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/403702306' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:15.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:14 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:15.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:14 smithi033 ceph-mon[96568]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T10:08:15.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:14 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/403702306' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:15.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:14 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:15.251 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.directory_methods (28 ms) 2024-09-21T10:08:15.251 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.create 2024-09-21T10:08:15.263 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.create (12 ms) 2024-09-21T10:08:15.263 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_features 2024-09-21T10:08:15.265 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2024-09-21T10:08:15.265 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-21T10:08:15.266 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_object_prefix (1 ms) 2024-09-21T10:08:15.266 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-21T10:08:15.268 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-21T10:08:15.269 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-21T10:08:15.270 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2024-09-21T10:08:15.270 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-21T10:08:15.271 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-21T10:08:15.271 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-21T10:08:15.275 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2024-09-21T10:08:15.275 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.get_size 2024-09-21T10:08:15.279 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-21T10:08:15.279 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.set_size 2024-09-21T10:08:15.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:14 smithi105 ceph-mon[86649]: osdmap e979: 8 total, 8 up, 8 in 2024-09-21T10:08:15.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:14 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/403702306' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:15.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:14 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:15.283 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2024-09-21T10:08:15.283 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-21T10:08:15.300 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2024-09-21T10:08:15.300 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-21T10:08:15.308 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2024-09-21T10:08:15.309 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-21T10:08:15.347 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.parents_v1 (39 ms) 2024-09-21T10:08:15.348 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-21T10:08:15.376 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.parents_v2 (29 ms) 2024-09-21T10:08:15.377 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-21T10:08:15.399 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.snapshots (23 ms) 2024-09-21T10:08:15.399 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-21T10:08:15.407 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.snapid_race (8 ms) 2024-09-21T10:08:15.407 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-21T10:08:15.416 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.stripingv2 (9 ms) 2024-09-21T10:08:15.416 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-21T10:08:15.419 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.object_map_save (3 ms) 2024-09-21T10:08:15.419 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-21T10:08:15.427 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2024-09-21T10:08:15.427 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-21T10:08:15.435 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.object_map_update (7 ms) 2024-09-21T10:08:15.435 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-21T10:08:15.435 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.object_map_load_enoent (1 ms) 2024-09-21T10:08:15.435 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-21T10:08:15.445 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.object_map_snap_add (10 ms) 2024-09-21T10:08:15.445 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-21T10:08:15.453 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2024-09-21T10:08:15.454 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.flags 2024-09-21T10:08:15.458 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-09-21T10:08:15.459 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.metadata 2024-09-21T10:08:15.488 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.metadata (30 ms) 2024-09-21T10:08:15.488 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.set_features 2024-09-21T10:08:15.495 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2024-09-21T10:08:15.495 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.mirror 2024-09-21T10:08:15.527 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.mirror (33 ms) 2024-09-21T10:08:15.528 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-21T10:08:15.541 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.mirror_image (14 ms) 2024-09-21T10:08:15.541 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-21T10:08:16.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[96568]: pgmap v1142: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:16.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[96568]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:16.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:16.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[96568]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T10:08:16.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:08:16.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:16.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[98546]: pgmap v1142: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:16.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[98546]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:16.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:16.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[98546]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T10:08:16.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:08:16.247 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:15 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:15 smithi105 ceph-mon[86649]: pgmap v1142: 137 pgs: 3 creating+peering, 29 unknown, 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:15 smithi105 ceph-mon[86649]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-21T10:08:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:15 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi033-130107-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:15 smithi105 ceph-mon[86649]: osdmap e980: 8 total, 8 up, 8 in 2024-09-21T10:08:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:15 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:08:16.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:15 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:17.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:16 smithi105 ceph-mon[86649]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T10:08:17.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:16 smithi033 ceph-mon[96568]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T10:08:17.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:16 smithi033 ceph-mon[98546]: osdmap e981: 8 total, 8 up, 8 in 2024-09-21T10:08:17.759 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.mirror_image_status (2218 ms) 2024-09-21T10:08:17.759 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-21T10:08:17.847 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.mirror_image_map (88 ms) 2024-09-21T10:08:17.847 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-21T10:08:17.855 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2024-09-21T10:08:17.856 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-21T10:08:17.867 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.mirror_snapshot (12 ms) 2024-09-21T10:08:17.867 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-21T10:08:17.870 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2024-09-21T10:08:17.870 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-21T10:08:17.873 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_dir_add (1 ms) 2024-09-21T10:08:17.873 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-21T10:08:17.874 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-21T10:08:17.874 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-21T10:08:17.880 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-09-21T10:08:17.880 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-21T10:08:17.883 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-21T10:08:17.883 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-21T10:08:17.887 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2024-09-21T10:08:17.887 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-21T10:08:17.889 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-21T10:08:17.889 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-21T10:08:17.892 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2024-09-21T10:08:17.892 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-21T10:08:17.895 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-21T10:08:17.895 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-21T10:08:17.899 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-21T10:08:17.899 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-21T10:08:17.901 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2024-09-21T10:08:17.901 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-21T10:08:17.905 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-21T10:08:17.905 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-21T10:08:17.907 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-21T10:08:17.907 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-21T10:08:17.909 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-21T10:08:17.909 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-21T10:08:17.911 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-21T10:08:17.911 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-21T10:08:17.914 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2024-09-21T10:08:17.914 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-21T10:08:17.916 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-09-21T10:08:17.916 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-21T10:08:17.919 INFO:tasks.workunit.client.0.smithi033.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1649-gfbd1a6ab/rpm/el9/BUILD/ceph-17.2.7-1649-gfbd1a6ab/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-21T10:08:17.919 INFO:tasks.workunit.client.0.smithi033.stdout:Expected equality of these values: 2024-09-21T10:08:17.919 INFO:tasks.workunit.client.0.smithi033.stdout: 1U 2024-09-21T10:08:17.919 INFO:tasks.workunit.client.0.smithi033.stdout: Which is: 1 2024-09-21T10:08:17.919 INFO:tasks.workunit.client.0.smithi033.stdout: keys.size() 2024-09-21T10:08:17.919 INFO:tasks.workunit.client.0.smithi033.stdout: Which is: 3 2024-09-21T10:08:17.919 INFO:tasks.workunit.client.0.smithi033.stdout:[ FAILED ] TestClsRbd.group_snap_set (3 ms) 2024-09-21T10:08:17.919 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-21T10:08:17.922 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-21T10:08:17.922 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-21T10:08:17.938 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (16 ms) 2024-09-21T10:08:17.938 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-21T10:08:18.141 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (203 ms) 2024-09-21T10:08:18.142 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-21T10:08:18.144 INFO:tasks.workunit.client.0.smithi033.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/17.2.7-1649-gfbd1a6ab/rpm/el9/BUILD/ceph-17.2.7-1649-gfbd1a6ab/src/test/cls_rbd/test_cls_rbd.cc:2792: Failure 2024-09-21T10:08:18.144 INFO:tasks.workunit.client.0.smithi033.stdout:Expected equality of these values: 2024-09-21T10:08:18.144 INFO:tasks.workunit.client.0.smithi033.stdout: 1U 2024-09-21T10:08:18.144 INFO:tasks.workunit.client.0.smithi033.stdout: Which is: 1 2024-09-21T10:08:18.144 INFO:tasks.workunit.client.0.smithi033.stdout: keys.size() 2024-09-21T10:08:18.144 INFO:tasks.workunit.client.0.smithi033.stdout: Which is: 3 2024-09-21T10:08:18.144 INFO:tasks.workunit.client.0.smithi033.stdout:[ FAILED ] TestClsRbd.group_snap_remove (3 ms) 2024-09-21T10:08:18.144 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-21T10:08:18.146 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2024-09-21T10:08:18.146 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-21T10:08:18.152 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.trash_methods (6 ms) 2024-09-21T10:08:18.152 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.op_features 2024-09-21T10:08:18.159 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2024-09-21T10:08:18.160 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-21T10:08:18.179 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2024-09-21T10:08:18.179 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-21T10:08:18.187 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-21T10:08:18.187 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-21T10:08:18.198 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2024-09-21T10:08:18.198 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-21T10:08:18.205 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2024-09-21T10:08:18.205 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.migration 2024-09-21T10:08:18.216 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.migration (11 ms) 2024-09-21T10:08:18.216 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-21T10:08:18.227 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.migration_v1 (11 ms) 2024-09-21T10:08:18.227 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-21T10:08:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:18 smithi105 ceph-mon[86649]: pgmap v1145: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s rd, 19 KiB/s wr, 64 op/s 2024-09-21T10:08:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:18 smithi105 ceph-mon[86649]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:18.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:18 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:08:18.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:18 smithi033 ceph-mon[98546]: pgmap v1145: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s rd, 19 KiB/s wr, 64 op/s 2024-09-21T10:08:18.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:18 smithi033 ceph-mon[98546]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:18.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:18 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:08:18.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:18 smithi033 ceph-mon[96568]: pgmap v1145: 137 pgs: 6 creating+peering, 15 unknown, 116 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s rd, 19 KiB/s wr, 64 op/s 2024-09-21T10:08:18.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:18 smithi033 ceph-mon[96568]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-21T10:08:18.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:18 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:08:20.013 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1786 ms) 2024-09-21T10:08:20.014 INFO:tasks.workunit.client.0.smithi033.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-21T10:08:20.058 INFO:tasks.workunit.client.0.smithi033.stdout:[ OK ] TestClsRbd.sparsify (45 ms) 2024-09-21T10:08:20.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:20 smithi105 ceph-mon[86649]: pgmap v1146: 137 pgs: 6 creating+peering, 7 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 18 KiB/s wr, 69 op/s 2024-09-21T10:08:20.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:20 smithi105 ceph-mon[86649]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T10:08:20.280 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:08:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:08:20.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:20 smithi033 ceph-mon[98546]: pgmap v1146: 137 pgs: 6 creating+peering, 7 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 18 KiB/s wr, 69 op/s 2024-09-21T10:08:20.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:20 smithi033 ceph-mon[98546]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T10:08:20.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:20 smithi033 ceph-mon[96568]: pgmap v1146: 137 pgs: 6 creating+peering, 7 unknown, 124 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 18 KiB/s wr, 69 op/s 2024-09-21T10:08:20.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:20 smithi033 ceph-mon[96568]: osdmap e982: 8 total, 8 up, 8 in 2024-09-21T10:08:21.034 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] 70 tests from TestClsRbd (5071 ms total) 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout:[----------] Global test environment tear-down 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout:[==========] 70 tests from 1 test suite ran. (7903 ms total) 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout:[ PASSED ] 68 tests. 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout:[ FAILED ] 2 tests, listed below: 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout:[ FAILED ] TestClsRbd.group_snap_remove 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout: 2024-09-21T10:08:21.035 INFO:tasks.workunit.client.0.smithi033.stdout: 2 FAILED TESTS 2024-09-21T10:08:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:08:21.038 INFO:tasks.workunit:Stopping ['cls'] on client.0... 2024-09-21T10:08:21.038 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-21T10:08:21.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:21 smithi105 ceph-mon[86649]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T10:08:21.304 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:21 smithi033 ceph-mon[98546]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T10:08:21.305 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:21 smithi033 ceph-mon[96568]: osdmap e983: 8 total, 8 up, 8 in 2024-09-21T10:08:21.306 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi033 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T10:08:21.541 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=6b75b9c430ac492ca49a3a1425b6cf0b Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/full_sequential.py", line 30, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_62da54b4b83a96ca9976b7e1d0c8272a564a2208/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi033 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-21T10:08:21.544 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-21T10:08:21.559 INFO:tasks.cephadm:Teardown begin 2024-09-21T10:08:21.560 INFO:tasks.cephfs.filesystem:Destroying file system cephfs and related pools 2024-09-21T10:08:21.560 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T10:08:21.821 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:21.943 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:21 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:08:21] "GET /metrics HTTP/1.1" 200 36453 "" "Prometheus/2.51.0" 2024-09-21T10:08:22.228 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:22 smithi033 ceph-mon[98546]: pgmap v1149: 137 pgs: 137 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 2.48k op/s 2024-09-21T10:08:22.228 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:22 smithi033 ceph-mon[98546]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T10:08:22.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:22 smithi033 ceph-mon[96568]: pgmap v1149: 137 pgs: 137 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 2.48k op/s 2024-09-21T10:08:22.229 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:22 smithi033 ceph-mon[96568]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T10:08:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:22 smithi105 ceph-mon[86649]: pgmap v1149: 137 pgs: 137 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 1.5 MiB/s wr, 2.48k op/s 2024-09-21T10:08:22.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:22 smithi105 ceph-mon[86649]: osdmap e984: 8 total, 8 up, 8 in 2024-09-21T10:08:23.726 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T10:08:23.726 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":15,"btime":"2024-09-21T10:00:18:529966+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54352,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2342313412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2342313412},{"type":"v1","addr":"172.21.15.105:6833","nonce":2342313412}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":15}],"filesystems":[{"mdsmap":{"epoch":14,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:47:25.959476+0000","modified":"2024-09-21T10:00:12.724476+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":581,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":44441},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44441":{"gid":44441,"name":"a","rank":0,"incarnation":11,"state":"up:active","state_seq":6,"addr":"172.21.15.33:6835/1626081805","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":1626081805},{"type":"v1","addr":"172.21.15.33:6835","nonce":1626081805}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":44441,"qdb_cluster":[44441]},"id":1}]} 2024-09-21T10:08:23.726 INFO:teuthology.orchestra.run.smithi033.stderr:dumped fsmap epoch 15 2024-09-21T10:08:24.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:24 smithi105 ceph-mon[86649]: pgmap v1151: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:24.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:24 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2204494699' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T10:08:24.356 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:24 smithi033 ceph-mon[96568]: pgmap v1151: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:24.356 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:24 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2204494699' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T10:08:24.356 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:24 smithi033 ceph-mon[98546]: pgmap v1151: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:24.356 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:24 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2204494699' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T10:08:24.388 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs dump --format=json 2024-09-21T10:08:24.653 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:26.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:26 smithi105 ceph-mon[86649]: pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:26.372 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:26 smithi033 ceph-mon[98546]: pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:26.373 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:26 smithi033 ceph-mon[96568]: pgmap v1152: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:26.697 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T10:08:26.697 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":15,"btime":"2024-09-21T10:00:18:529966+0000","default_fscid":1,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2"}},"feature_flags":{"enable_multiple":true,"ever_enabled_multiple":true},"standbys":[{"gid":54352,"name":"b","rank":-1,"incarnation":0,"state":"up:standby","state_seq":1,"addr":"172.21.15.105:6833/2342313412","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6832","nonce":2342313412},{"type":"v1","addr":"172.21.15.105:6833","nonce":2342313412}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"epoch":15}],"filesystems":[{"mdsmap":{"epoch":14,"flags":18,"flags_state":{"joinable":true,"allow_snaps":true,"allow_multimds_snaps":true,"allow_standby_replay":false,"refuse_client_session":false,"refuse_standby_for_another_fs":false,"balance_automate":false},"ever_allowed_features":0,"explicitly_allowed_features":0,"created":"2024-09-21T09:47:25.959476+0000","modified":"2024-09-21T10:00:12.724476+0000","tableserver":0,"root":0,"session_timeout":60,"session_autoclose":300,"required_client_features":{},"max_file_size":1099511627776,"max_xattr_size":65536,"last_failure":0,"last_failure_osd_epoch":581,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}},"max_mds":1,"in":[0],"up":{"mds_0":44441},"failed":[],"damaged":[],"stopped":[],"info":{"gid_44441":{"gid":44441,"name":"a","rank":0,"incarnation":11,"state":"up:active","state_seq":6,"addr":"172.21.15.33:6835/1626081805","addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6834","nonce":1626081805},{"type":"v1","addr":"172.21.15.33:6835","nonce":1626081805}]},"join_fscid":-1,"export_targets":[],"features":4540719139924082687,"flags":0,"compat":{"compat":{},"ro_compat":{},"incompat":{"feature_1":"base v0.20","feature_2":"client writeable ranges","feature_3":"default file layouts on dirs","feature_4":"dir inode in separate object","feature_5":"mds uses versioned encoding","feature_6":"dirfrag is stored in omap","feature_7":"mds uses inline data","feature_8":"no anchor table","feature_9":"file layout v2","feature_10":"snaprealm v2","feature_11":"minor log segments","feature_12":"quiesce subvolumes"}}}},"data_pools":[3],"metadata_pool":2,"enabled":true,"fs_name":"cephfs","balancer":"","bal_rank_mask":"-1","standby_count_wanted":0,"qdb_leader":44441,"qdb_cluster":[44441]},"id":1}]} 2024-09-21T10:08:26.697 INFO:teuthology.orchestra.run.smithi033.stderr:dumped fsmap epoch 15 2024-09-21T10:08:27.326 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:27 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1595675914' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T10:08:27.326 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:27 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1595675914' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T10:08:27.352 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd dump --format=json 2024-09-21T10:08:27.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:27 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1595675914' entity='client.admin' cmd={"prefix": "fs dump", "format": "json"} : dispatch 2024-09-21T10:08:27.608 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:28.496 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:28 smithi033 ceph-mon[98546]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:28.496 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:28 smithi033 ceph-mon[96568]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:28.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:28 smithi105 ceph-mon[86649]: pgmap v1153: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:29.521 INFO:teuthology.orchestra.run.smithi033.stdout: 2024-09-21T10:08:29.521 INFO:teuthology.orchestra.run.smithi033.stdout:{"epoch":984,"fsid":"a29e8460-77fd-11ef-baf6-efdc52797490","created":"2024-09-21T09:41:51.424202+0000","modified":"2024-09-21T10:08:21.014060+0000","last_up_change":"2024-09-21T09:59:49.901411+0000","last_in_change":"2024-09-21T09:46:38.429366+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":60,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-21T09:44:57.819538+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"cephfs_metadata","create_time":"2024-09-21T09:47:19.790061+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs_data","create_time":"2024-09-21T09:47:22.976967+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":64,"pg_placement_num":64,"pg_placement_num_target":64,"pg_num_target":64,"pg_num_pending":64,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_min":64,"target_size_ratio":0.89999997615814209},"application_metadata":{"cephfs":{"data":"cephfs"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.3799999952316284,"score_stable":1.3799999952316284,"optimal_score":1,"raw_score_acting":1.3799999952316284,"raw_score_stable":1.3799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"rbd","create_time":"2024-09-21T09:48:32.209917+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":3,"score_stable":3,"optimal_score":1,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"27f5a40d-6f81-49ea-a8aa-0641d32c3667","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":312,"up_from":325,"up_thru":979,"down_at":313,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6802","nonce":189402874},{"type":"v1","addr":"172.21.15.33:6803","nonce":189402874}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6804","nonce":189402874},{"type":"v1","addr":"172.21.15.33:6805","nonce":189402874}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6808","nonce":189402874},{"type":"v1","addr":"172.21.15.33:6809","nonce":189402874}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6806","nonce":189402874},{"type":"v1","addr":"172.21.15.33:6807","nonce":189402874}]},"public_addr":"172.21.15.33:6803/189402874","cluster_addr":"172.21.15.33:6805/189402874","heartbeat_back_addr":"172.21.15.33:6809/189402874","heartbeat_front_addr":"172.21.15.33:6807/189402874","state":["exists","up"]},{"osd":1,"uuid":"120c7205-c87f-4d77-a255-14fff1e3b631","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":346,"up_from":359,"up_thru":979,"down_at":347,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6810","nonce":2766504800},{"type":"v1","addr":"172.21.15.33:6811","nonce":2766504800}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6812","nonce":2766504800},{"type":"v1","addr":"172.21.15.33:6813","nonce":2766504800}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6816","nonce":2766504800},{"type":"v1","addr":"172.21.15.33:6817","nonce":2766504800}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6814","nonce":2766504800},{"type":"v1","addr":"172.21.15.33:6815","nonce":2766504800}]},"public_addr":"172.21.15.33:6811/2766504800","cluster_addr":"172.21.15.33:6813/2766504800","heartbeat_back_addr":"172.21.15.33:6817/2766504800","heartbeat_front_addr":"172.21.15.33:6815/2766504800","state":["exists","up"]},{"osd":2,"uuid":"bf7f6b7d-0134-4ce6-b776-040aa14b9075","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":380,"up_from":393,"up_thru":979,"down_at":381,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6818","nonce":586780654},{"type":"v1","addr":"172.21.15.33:6819","nonce":586780654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6820","nonce":586780654},{"type":"v1","addr":"172.21.15.33:6821","nonce":586780654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6824","nonce":586780654},{"type":"v1","addr":"172.21.15.33:6825","nonce":586780654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6822","nonce":586780654},{"type":"v1","addr":"172.21.15.33:6823","nonce":586780654}]},"public_addr":"172.21.15.33:6819/586780654","cluster_addr":"172.21.15.33:6821/586780654","heartbeat_back_addr":"172.21.15.33:6825/586780654","heartbeat_front_addr":"172.21.15.33:6823/586780654","state":["exists","up"]},{"osd":3,"uuid":"697c4c8e-f54d-4cfe-8dfd-a7c591a94a87","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":413,"up_from":426,"up_thru":979,"down_at":414,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6826","nonce":610865334},{"type":"v1","addr":"172.21.15.33:6827","nonce":610865334}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6828","nonce":610865334},{"type":"v1","addr":"172.21.15.33:6829","nonce":610865334}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6832","nonce":610865334},{"type":"v1","addr":"172.21.15.33:6833","nonce":610865334}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.33:6830","nonce":610865334},{"type":"v1","addr":"172.21.15.33:6831","nonce":610865334}]},"public_addr":"172.21.15.33:6827/610865334","cluster_addr":"172.21.15.33:6829/610865334","heartbeat_back_addr":"172.21.15.33:6833/610865334","heartbeat_front_addr":"172.21.15.33:6831/610865334","state":["exists","up"]},{"osd":4,"uuid":"3040cdba-b341-48d0-bdf0-c6b6655fb164","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":446,"up_from":460,"up_thru":979,"down_at":447,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6800","nonce":2250129110},{"type":"v1","addr":"172.21.15.105:6801","nonce":2250129110}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6802","nonce":2250129110},{"type":"v1","addr":"172.21.15.105:6803","nonce":2250129110}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6806","nonce":2250129110},{"type":"v1","addr":"172.21.15.105:6807","nonce":2250129110}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6804","nonce":2250129110},{"type":"v1","addr":"172.21.15.105:6805","nonce":2250129110}]},"public_addr":"172.21.15.105:6801/2250129110","cluster_addr":"172.21.15.105:6803/2250129110","heartbeat_back_addr":"172.21.15.105:6807/2250129110","heartbeat_front_addr":"172.21.15.105:6805/2250129110","state":["exists","up"]},{"osd":5,"uuid":"0e621bf2-e921-45ab-8ee8-18b44317c562","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":36,"last_clean_end":480,"up_from":494,"up_thru":979,"down_at":481,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6808","nonce":1966612698},{"type":"v1","addr":"172.21.15.105:6809","nonce":1966612698}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6810","nonce":1966612698},{"type":"v1","addr":"172.21.15.105:6811","nonce":1966612698}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6814","nonce":1966612698},{"type":"v1","addr":"172.21.15.105:6815","nonce":1966612698}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6812","nonce":1966612698},{"type":"v1","addr":"172.21.15.105:6813","nonce":1966612698}]},"public_addr":"172.21.15.105:6809/1966612698","cluster_addr":"172.21.15.105:6811/1966612698","heartbeat_back_addr":"172.21.15.105:6815/1966612698","heartbeat_front_addr":"172.21.15.105:6813/1966612698","state":["exists","up"]},{"osd":6,"uuid":"a57f3528-3cfb-4319-bf62-fa3a0762cfe4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":42,"last_clean_end":512,"up_from":525,"up_thru":979,"down_at":513,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6816","nonce":580557017},{"type":"v1","addr":"172.21.15.105:6817","nonce":580557017}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6818","nonce":580557017},{"type":"v1","addr":"172.21.15.105:6819","nonce":580557017}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6822","nonce":580557017},{"type":"v1","addr":"172.21.15.105:6823","nonce":580557017}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6820","nonce":580557017},{"type":"v1","addr":"172.21.15.105:6821","nonce":580557017}]},"public_addr":"172.21.15.105:6817/580557017","cluster_addr":"172.21.15.105:6819/580557017","heartbeat_back_addr":"172.21.15.105:6823/580557017","heartbeat_front_addr":"172.21.15.105:6821/580557017","state":["exists","up"]},{"osd":7,"uuid":"cbbae365-68d8-4e30-bf12-b1049489cb50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":48,"last_clean_end":547,"up_from":560,"up_thru":979,"down_at":548,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6824","nonce":3020670744},{"type":"v1","addr":"172.21.15.105:6825","nonce":3020670744}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6826","nonce":3020670744},{"type":"v1","addr":"172.21.15.105:6827","nonce":3020670744}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6830","nonce":3020670744},{"type":"v1","addr":"172.21.15.105:6831","nonce":3020670744}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.105:6828","nonce":3020670744},{"type":"v1","addr":"172.21.15.105:6829","nonce":3020670744}]},"public_addr":"172.21.15.105:6825/3020670744","cluster_addr":"172.21.15.105:6827/3020670744","heartbeat_back_addr":"172.21.15.105:6831/3020670744","heartbeat_front_addr":"172.21.15.105:6829/3020670744","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:08.106168+0000","dead_epoch":312},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:31.866232+0000","dead_epoch":345},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:44:54.791169+0000","dead_epoch":380},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:18.160495+0000","dead_epoch":413},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:45:40.534590+0000","dead_epoch":446},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:04.722993+0000","dead_epoch":480},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:29.687072+0000","dead_epoch":512},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-21T09:46:53.881602+0000","dead_epoch":547}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"2.1a","mappings":[{"from":7,"to":2}]},{"pgid":"3.b","mappings":[{"from":0,"to":2}]},{"pgid":"3.1c","mappings":[{"from":5,"to":2}]},{"pgid":"3.2f","mappings":[{"from":4,"to":3}]},{"pgid":"3.34","mappings":[{"from":4,"to":2}]},{"pgid":"3.39","mappings":[{"from":0,"to":1}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.105:6833/2691139879":"2024-09-22T10:00:09.714128+0000","172.21.15.33:0/2645856759":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/2290444761":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/3865133777":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/3947635653":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/150268354":"2024-09-22T09:48:29.268285+0000","172.21.15.105:6835/3560340417":"2024-09-22T09:53:50.171580+0000","172.21.15.105:6834/3560340417":"2024-09-22T09:53:50.171580+0000","172.21.15.33:0/482369733":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/131382457":"2024-09-22T09:52:46.436950+0000","172.21.15.105:0/1108084292":"2024-09-22T09:53:50.171580+0000","172.21.15.33:0/4220506658":"2024-09-22T09:54:30.283006+0000","172.21.15.33:6800/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:6801/244283938":"2024-09-22T09:48:29.268285+0000","172.21.15.33:6800/244283938":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/1301914674":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/3680291673":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6801/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:6800/2236080153":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/2041164136":"2024-09-22T09:52:46.436950+0000","172.21.15.105:0/1100101125":"2024-09-22T09:53:50.171580+0000","172.21.15.33:0/3579553953":"2024-09-22T09:52:46.436950+0000","172.21.15.33:0/3782954164":"2024-09-22T09:42:45.324074+0000","172.21.15.33:0/700678975":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/2740931131":"2024-09-22T09:54:30.283006+0000","172.21.15.33:6801/2967063367":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/2390585401":"2024-09-22T09:52:46.436950+0000","172.21.15.33:6800/2967063367":"2024-09-22T09:54:30.283006+0000","172.21.15.33:0/395059673":"2024-09-22T09:52:46.436950+0000","172.21.15.33:6800/289485480":"2024-09-22T09:52:46.436950+0000","172.21.15.33:0/1367140190":"2024-09-22T09:42:21.098281+0000","172.21.15.105:0/2004410743":"2024-09-22T09:53:50.171580+0000","172.21.15.33:6801/289485480":"2024-09-22T09:52:46.436950+0000","172.21.15.105:0/3260429203":"2024-09-22T09:53:50.171580+0000","172.21.15.105:0/253870186":"2024-09-22T09:53:50.171580+0000","172.21.15.33:0/2876636439":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/1606444157":"2024-09-22T09:54:30.283006+0000","172.21.15.105:6832/2691139879":"2024-09-22T10:00:09.714128+0000","172.21.15.33:6801/2389836358":"2024-09-22T09:42:21.098281+0000","172.21.15.33:0/1156071037":"2024-09-22T09:48:29.268285+0000","172.21.15.33:0/3347076479":"2024-09-22T09:48:29.268285+0000","172.21.15.105:0/768914160":"2024-09-22T09:53:50.171580+0000","172.21.15.105:0/4091807878":"2024-09-22T09:53:50.171580+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"},"teuthologyprofile":{"crush-device-class":"","crush-failure-domain":"osd","crush-root":"default","jerasure-per-chunk-alignment":"false","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van","w":"8"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":434,"length":1}]},{"pool":60,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-21T10:08:30.147 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:30 smithi033 ceph-mon[98546]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:30.147 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:30 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1577145097' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:08:30.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:30 smithi033 ceph-mon[96568]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:30.148 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:30 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1577145097' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:08:30.173 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs fail cephfs 2024-09-21T10:08:30.332 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:30 smithi105 ceph-mon[86649]: pgmap v1154: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:30.333 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:30 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1577145097' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-21T10:08:30.333 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:08:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:08:30.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:30.430 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:31.871 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:31 smithi033 ceph-mon[98546]: pgmap v1155: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:31.871 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:08:31.871 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:31 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:31.872 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:31 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:08:31] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.51.0" 2024-09-21T10:08:31.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:31 smithi033 ceph-mon[96568]: pgmap v1155: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:31.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:08:31.872 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:31 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:31 smithi105 ceph-mon[86649]: pgmap v1155: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' 2024-09-21T10:08:32.030 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:31 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-21T10:08:32.571 INFO:teuthology.orchestra.run.smithi033.stderr:cephfs marked not joinable; MDS cannot join the cluster. All MDS ranks marked failed. 2024-09-21T10:08:32.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1652238190' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-21T10:08:32.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[98546]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:08:32.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[98546]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:08:32.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/1652238190' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-21T10:08:32.897 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[98546]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T10:08:32.898 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[98546]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T10:08:32.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a[96545]: 2024-09-21T10:08:32.568+0000 7fadcba59640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:08:32.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1652238190' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-21T10:08:32.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[96568]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:08:32.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[96568]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:08:32.898 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/1652238190' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-21T10:08:32.899 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[96568]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T10:08:32.899 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-mon[96568]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T10:08:32.899 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[123139]: -4083> 2024-09-21T10:00:14.984+0000 7f83a6913640 -1 mds.pinger is_rank_lagging: rank=0 was never sent ping request. 2024-09-21T10:08:32.899 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[123139]: ignoring --setuser ceph since I am not root 2024-09-21T10:08:32.899 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[123139]: ignoring --setgroup ceph since I am not root 2024-09-21T10:08:32.899 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:08:32 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[123139]: starting mds.a at 2024-09-21T10:08:33.199 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph fs rm cephfs --yes-i-really-mean-it 2024-09-21T10:08:33.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:32 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1652238190' entity='client.admin' cmd={"prefix": "fs fail", "fs_name": "cephfs"} : dispatch 2024-09-21T10:08:33.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:32 smithi105 ceph-mon[86649]: Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-21T10:08:33.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:32 smithi105 ceph-mon[86649]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-21T10:08:33.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:32 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/1652238190' entity='client.admin' cmd='[{"prefix": "fs fail", "fs_name": "cephfs"}]': finished 2024-09-21T10:08:33.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:32 smithi105 ceph-mon[86649]: osdmap e985: 8 total, 8 up, 8 in 2024-09-21T10:08:33.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:32 smithi105 ceph-mon[86649]: fsmap cephfs:0/1 1 up:standby, 1 failed 2024-09-21T10:08:33.459 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:33.839 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:33 smithi033 ceph-mon[96568]: pgmap v1156: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:33.839 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:33 smithi033 ceph-mon[96568]: mds.? [v2:172.21.15.33:6834/1703284702,v1:172.21.15.33:6835/1703284702] up:boot 2024-09-21T10:08:33.840 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:33 smithi033 ceph-mon[96568]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-21T10:08:33.840 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:33 smithi033 ceph-mon[96568]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T10:08:33.840 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:33 smithi033 ceph-mon[98546]: pgmap v1156: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:33.840 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:33 smithi033 ceph-mon[98546]: mds.? [v2:172.21.15.33:6834/1703284702,v1:172.21.15.33:6835/1703284702] up:boot 2024-09-21T10:08:33.840 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:33 smithi033 ceph-mon[98546]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-21T10:08:33.841 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:33 smithi033 ceph-mon[98546]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T10:08:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:33 smithi105 ceph-mon[86649]: pgmap v1156: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:34.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:33 smithi105 ceph-mon[86649]: mds.? [v2:172.21.15.33:6834/1703284702,v1:172.21.15.33:6835/1703284702] up:boot 2024-09-21T10:08:34.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:33 smithi105 ceph-mon[86649]: fsmap cephfs:0/1 2 up:standby, 1 failed 2024-09-21T10:08:34.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:33 smithi105 ceph-mon[86649]: from='mgr.25157 172.21.15.33:0/3279967078' entity='mgr.y' cmd={"prefix": "mds metadata", "who": "a"} : dispatch 2024-09-21T10:08:36.083 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:35 smithi033 ceph-mon[96568]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:36.083 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:35 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3165489384' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:36.083 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:35 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:36.083 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:35 smithi033 ceph-mon[98546]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:36.083 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:35 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3165489384' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:36.083 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:35 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:35 smithi105 ceph-mon[86649]: pgmap v1158: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:36.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:35 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3165489384' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:36.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:35 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd={"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true} : dispatch 2024-09-21T10:08:36.356 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm cephfs_metadata cephfs_metadata --yes-i-really-really-mean-it 2024-09-21T10:08:36.619 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[98546]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:08:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[98546]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:08:36.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[98546]: Cluster is now healthy 2024-09-21T10:08:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[98546]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:36.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[98546]: fsmap 2 up:standby 2024-09-21T10:08:36.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[96568]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:08:36.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[96568]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:08:36.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[96568]: Cluster is now healthy 2024-09-21T10:08:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[96568]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:36.998 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:36 smithi033 ceph-mon[96568]: fsmap 2 up:standby 2024-09-21T10:08:37.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:36 smithi105 ceph-mon[86649]: Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-21T10:08:37.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:36 smithi105 ceph-mon[86649]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-21T10:08:37.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:36 smithi105 ceph-mon[86649]: Cluster is now healthy 2024-09-21T10:08:37.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:36 smithi105 ceph-mon[86649]: from='client.? ' entity='client.admin' cmd='[{"prefix": "fs rm", "fs_name": "cephfs", "yes_i_really_mean_it": true}]': finished 2024-09-21T10:08:37.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:36 smithi105 ceph-mon[86649]: fsmap 2 up:standby 2024-09-21T10:08:37.996 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:37 smithi033 ceph-mon[98546]: pgmap v1159: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:37.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:37 smithi033 ceph-mon[96568]: pgmap v1159: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:38.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:37 smithi105 ceph-mon[86649]: pgmap v1159: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:38.811 INFO:teuthology.orchestra.run.smithi033.stderr:pool 'cephfs_metadata' removed 2024-09-21T10:08:39.246 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:38 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2521037829' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:39.246 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:38 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2521037829' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:39.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:38 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2521037829' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:39.666 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a29e8460-77fd-11ef-baf6-efdc52797490 -- ceph osd pool rm cephfs_data cephfs_data --yes-i-really-really-mean-it 2024-09-21T10:08:39.926 INFO:teuthology.orchestra.run.smithi033.stderr:Inferring config /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/mon.a/config 2024-09-21T10:08:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:39 smithi033 ceph-mon[96568]: pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:39 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/2521037829' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:39.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:39 smithi033 ceph-mon[96568]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T10:08:39.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:39 smithi033 ceph-mon[98546]: pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:39.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:39 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/2521037829' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:39.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:39 smithi033 ceph-mon[98546]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T10:08:40.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:39 smithi105 ceph-mon[86649]: pgmap v1160: 105 pgs: 105 active+clean; 583 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:40.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:39 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/2521037829' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_metadata", "pool2": "cephfs_metadata", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:40.215 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:39 smithi105 ceph-mon[86649]: osdmap e986: 8 total, 8 up, 8 in 2024-09-21T10:08:40.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:08:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:08:40.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:41.814 INFO:teuthology.orchestra.run.smithi033.stderr:pool 'cephfs_data' removed 2024-09-21T10:08:41.996 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:41 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: ::ffff:172.21.15.105 - - [21/Sep/2024:10:08:41] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.51.0" 2024-09-21T10:08:41.996 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:41 smithi033 ceph-mon[96568]: pgmap v1162: 73 pgs: 73 active+clean; 577 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:41.997 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:41 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3782930812' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:41.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:41 smithi033 ceph-mon[98546]: pgmap v1162: 73 pgs: 73 active+clean; 577 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:41.997 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:41 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3782930812' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:41 smithi105 ceph-mon[86649]: pgmap v1162: 73 pgs: 73 active+clean; 577 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-21T10:08:42.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:41 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3782930812' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true} : dispatch 2024-09-21T10:08:42.511 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:08:42.541 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:08:42.574 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-21T10:08:42.574 DEBUG:teuthology.orchestra.run.smithi033:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:08:42.597 DEBUG:teuthology.orchestra.run.smithi105:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-21T10:08:42.631 INFO:tasks.cephadm:Stopping all daemons... 2024-09-21T10:08:42.631 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-21T10:08:42.631 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a 2024-09-21T10:08:42.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:42 smithi033 systemd[1]: Stopping Ceph mon.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:08:43.122 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:42 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:42] ENGINE Bus STOPPING 2024-09-21T10:08:43.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:42 smithi033 ceph-mon[98546]: from='client.? 172.21.15.33:0/3782930812' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:43.123 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:42 smithi033 ceph-mon[98546]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T10:08:43.124 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:42 smithi033 ceph-mon[96568]: from='client.? 172.21.15.33:0/3782930812' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:43.124 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:42 smithi033 ceph-mon[96568]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T10:08:43.124 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:42 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a[96545]: 2024-09-21T10:08:42.955+0000 7fadd1264640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:08:43.124 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:42 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a[96545]: 2024-09-21T10:08:42.955+0000 7fadd1264640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-21T10:08:43.281 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:42 smithi105 ceph-mon[86649]: from='client.? 172.21.15.33:0/3782930812' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "cephfs_data", "pool2": "cephfs_data", "yes_i_really_really_mean_it": true}]': finished 2024-09-21T10:08:43.282 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:42 smithi105 ceph-mon[86649]: osdmap e987: 8 total, 8 up, 8 in 2024-09-21T10:08:43.481 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:43 smithi033 podman[131692]: 2024-09-21 10:08:43.233617182 +0000 UTC m=+0.401023629 container died 652442dcd6a4d173d70116d94b6b0d967af77acdda9e22815dc9bedd7eaf564d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-21T10:08:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T10:08:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:43] ENGINE Bus STOPPED 2024-09-21T10:08:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:43] ENGINE Bus STARTING 2024-09-21T10:08:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:43] ENGINE Serving on http://:::9283 2024-09-21T10:08:43.481 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:43 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:43] ENGINE Bus STARTED 2024-09-21T10:08:43.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:43 smithi033 podman[131692]: 2024-09-21 10:08:43.576998153 +0000 UTC m=+0.744404584 container cleanup 652442dcd6a4d173d70116d94b6b0d967af77acdda9e22815dc9bedd7eaf564d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-21T10:08:43.746 INFO:journalctl@ceph.mon.a.smithi033.stdout:Sep 21 10:08:43 smithi033 bash[131692]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-a 2024-09-21T10:08:44.013 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.a.service' 2024-09-21T10:08:44.057 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:08:44.058 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-21T10:08:44.058 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-21T10:08:44.058 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.c 2024-09-21T10:08:44.402 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:44 smithi033 systemd[1]: Stopping Ceph mon.c for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:08:44.667 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:44] ENGINE Bus STOPPING 2024-09-21T10:08:44.667 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c[98522]: 2024-09-21T10:08:44.399+0000 7f78d9adf640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:08:44.667 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c[98522]: 2024-09-21T10:08:44.399+0000 7f78d9adf640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-21T10:08:44.667 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:44 smithi033 podman[131835]: 2024-09-21 10:08:44.421147049 +0000 UTC m=+0.136813298 container died 5c9071ecac424d4a7b79d07a266b324d27ab517bf48bbf2c07d0cd2cedf4ae18 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-21T10:08:44.934 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:44 smithi033 podman[131835]: 2024-09-21 10:08:44.756567871 +0000 UTC m=+0.472234109 container cleanup 5c9071ecac424d4a7b79d07a266b324d27ab517bf48bbf2c07d0cd2cedf4ae18 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-21T10:08:44.934 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:44 smithi033 bash[131835]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c 2024-09-21T10:08:44.935 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-21T10:08:44.935 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:44] ENGINE Bus STOPPED 2024-09-21T10:08:44.935 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:44] ENGINE Bus STARTING 2024-09-21T10:08:44.935 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:44] ENGINE Serving on http://:::9283 2024-09-21T10:08:44.935 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:44 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: [21/Sep/2024:10:08:44] ENGINE Bus STARTED 2024-09-21T10:08:45.184 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.c.service' 2024-09-21T10:08:45.206 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:44 smithi033 podman[131852]: 2024-09-21 10:08:44.932755044 +0000 UTC m=+0.508236233 container remove 5c9071ecac424d4a7b79d07a266b324d27ab517bf48bbf2c07d0cd2cedf4ae18 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-09-21T10:08:45.206 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:45 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.c.service: Deactivated successfully. 2024-09-21T10:08:45.206 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:45 smithi033 systemd[1]: Stopped Ceph mon.c for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:08:45.206 INFO:journalctl@ceph.mon.c.smithi033.stdout:Sep 21 10:08:45 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.c.service: Consumed 12.209s CPU time. 2024-09-21T10:08:45.219 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:08:45.219 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-21T10:08:45.220 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-21T10:08:45.220 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.b 2024-09-21T10:08:45.530 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:45 smithi105 systemd[1]: Stopping Ceph mon.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:08:45.956 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b[86626]: 2024-09-21T10:08:45.528+0000 7f6fbbbdc640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:08:45.956 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b[86626]: 2024-09-21T10:08:45.528+0000 7f6fbbbdc640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-21T10:08:45.956 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:45 smithi105 podman[119053]: 2024-09-21 10:08:45.641457493 +0000 UTC m=+0.223515525 container died 9f90671a2e298396ed70122867b7702c03ce28b5680d5296540545d8285e6740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:08:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:46 smithi105 podman[119053]: 2024-09-21 10:08:46.127183618 +0000 UTC m=+0.709241692 container cleanup 9f90671a2e298396ed70122867b7702c03ce28b5680d5296540545d8285e6740 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T10:08:46.280 INFO:journalctl@ceph.mon.b.smithi105.stdout:Sep 21 10:08:46 smithi105 bash[119053]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mon-b 2024-09-21T10:08:46.547 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mon.b.service' 2024-09-21T10:08:46.591 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:08:46.591 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-21T10:08:46.591 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-21T10:08:46.591 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y 2024-09-21T10:08:46.906 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:46 smithi033 systemd[1]: Stopping Ceph mgr.y for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:08:47.189 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y[88550]: 2024-09-21T10:08:46.903+0000 7fc3ecbf3640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:08:47.189 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:46 smithi033 podman[131977]: 2024-09-21 10:08:46.937034213 +0000 UTC m=+0.141539580 container died 917c670c2752529c94adb0dd03344b77d50e24316c202e543fee1916845e2287 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T10:08:47.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:47 smithi033 podman[131977]: 2024-09-21 10:08:47.360850108 +0000 UTC m=+0.565355472 container cleanup 917c670c2752529c94adb0dd03344b77d50e24316c202e543fee1916845e2287 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-21T10:08:47.496 INFO:journalctl@ceph.mgr.y.smithi033.stdout:Sep 21 10:08:47 smithi033 bash[131977]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-y 2024-09-21T10:08:47.737 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.y.service' 2024-09-21T10:08:47.776 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:08:47.776 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-21T10:08:47.777 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-21T10:08:47.777 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x 2024-09-21T10:08:48.118 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 10:08:47 smithi105 systemd[1]: Stopping Ceph mgr.x for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:08:48.418 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 10:08:48 smithi105 podman[119186]: 2024-09-21 10:08:48.114556226 +0000 UTC m=+0.136816844 container died 5f55827751a5fb0133b4055ce05bbcc93689fd30fdd4599205c388cda652491e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731) 2024-09-21T10:08:48.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 10:08:48 smithi105 podman[119186]: 2024-09-21 10:08:48.506906688 +0000 UTC m=+0.529167316 container cleanup 5f55827751a5fb0133b4055ce05bbcc93689fd30fdd4599205c388cda652491e (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:08:48.780 INFO:journalctl@ceph.mgr.x.smithi105.stdout:Sep 21 10:08:48 smithi105 bash[119186]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mgr-x 2024-09-21T10:08:48.916 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mgr.x.service' 2024-09-21T10:08:48.950 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:08:48.950 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-21T10:08:48.950 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-21T10:08:48.951 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.0 2024-09-21T10:08:49.246 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:49 smithi033 systemd[1]: Stopping Ceph osd.0 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:08:49.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T10:08:49.266+0000 7f2c7945c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:08:49.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T10:08:49.266+0000 7f2c7945c640 -1 osd.0 987 *** Got signal Terminated *** 2024-09-21T10:08:49.747 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:49 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0[103843]: 2024-09-21T10:08:49.266+0000 7f2c7945c640 -1 osd.0 987 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:08:50.530 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:08:50 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:08:50.213Z caller=group.go:483 level=warn name=CephPoolGrowthWarning index=0 component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{cluster=\"a29e8460-77fd-11ef-baf6-efdc52797490\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-21T10:08:55.101 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:54 smithi033 podman[132122]: 2024-09-21 10:08:54.801890774 +0000 UTC m=+5.649008906 container died 28dc5f57b5bceaa29c8a5b97d30675e84972fa356f0949756fbb7daa467c94eb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T10:08:55.451 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:55 smithi033 podman[132122]: 2024-09-21 10:08:55.198227353 +0000 UTC m=+6.045345442 container cleanup 28dc5f57b5bceaa29c8a5b97d30675e84972fa356f0949756fbb7daa467c94eb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:08:55.452 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:55 smithi033 bash[132122]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0 2024-09-21T10:08:55.746 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:55 smithi033 podman[132145]: 2024-09-21 10:08:55.450025205 +0000 UTC m=+0.639245731 container remove 28dc5f57b5bceaa29c8a5b97d30675e84972fa356f0949756fbb7daa467c94eb (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0, org.label-schema.build-date=20240731, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:08:56.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:55 smithi033 podman[132240]: 2024-09-21 10:08:55.775657291 +0000 UTC m=+0.113101850 container create 0d31e5ba7d5549e13d364b0d9e8a240724a1c31103060c6907e515ce5239303b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-21T10:08:56.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:55 smithi033 podman[132240]: 2024-09-21 10:08:55.679033736 +0000 UTC m=+0.016478307 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:08:56.113 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:56 smithi033 podman[132240]: 2024-09-21 10:08:56.019754979 +0000 UTC m=+0.357199545 container init 0d31e5ba7d5549e13d364b0d9e8a240724a1c31103060c6907e515ce5239303b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=) 2024-09-21T10:08:56.114 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:56 smithi033 podman[132240]: 2024-09-21 10:08:56.024839782 +0000 UTC m=+0.362284338 container start 0d31e5ba7d5549e13d364b0d9e8a240724a1c31103060c6907e515ce5239303b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T10:08:56.114 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:56 smithi033 podman[132240]: 2024-09-21 10:08:56.111236035 +0000 UTC m=+0.448680608 container attach 0d31e5ba7d5549e13d364b0d9e8a240724a1c31103060c6907e515ce5239303b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:08:56.496 INFO:journalctl@ceph.osd.0.smithi033.stdout:Sep 21 10:08:56 smithi033 podman[132240]: 2024-09-21 10:08:56.218736645 +0000 UTC m=+0.556181213 container died 0d31e5ba7d5549e13d364b0d9e8a240724a1c31103060c6907e515ce5239303b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-62da54b) 2024-09-21T10:08:56.814 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.0.service' 2024-09-21T10:08:56.857 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:08:56.857 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-21T10:08:56.857 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-21T10:08:56.857 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.1 2024-09-21T10:08:57.239 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:08:56 smithi033 systemd[1]: Stopping Ceph osd.1 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:08:57.239 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:08:56 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:08:56.993Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:08:57.240 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:08:56 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:08:56.993Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:08:57.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:08:57 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[108464]: 2024-09-21T10:08:57.235+0000 7f420b705640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:08:57.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:08:57 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[108464]: 2024-09-21T10:08:57.235+0000 7f420b705640 -1 osd.1 987 *** Got signal Terminated *** 2024-09-21T10:08:57.496 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:08:57 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1[108464]: 2024-09-21T10:08:57.235+0000 7f420b705640 -1 osd.1 987 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:09:02.996 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:02 smithi033 podman[132368]: 2024-09-21 10:09:02.726912331 +0000 UTC m=+5.627145188 container died 1deebc8df60c3dda39de98090ae694322c515f61dead6b94491a8a143b1d3040 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:09:03.437 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:03 smithi033 podman[132368]: 2024-09-21 10:09:03.175884799 +0000 UTC m=+6.076117685 container cleanup 1deebc8df60c3dda39de98090ae694322c515f61dead6b94491a8a143b1d3040 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-21T10:09:03.437 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:03 smithi033 bash[132368]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1 2024-09-21T10:09:03.746 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:03 smithi033 podman[132389]: 2024-09-21 10:09:03.435433885 +0000 UTC m=+0.705318927 container remove 1deebc8df60c3dda39de98090ae694322c515f61dead6b94491a8a143b1d3040 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T10:09:04.073 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:03 smithi033 podman[132485]: 2024-09-21 10:09:03.717648802 +0000 UTC m=+0.016769936 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:09:04.074 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:03 smithi033 podman[132485]: 2024-09-21 10:09:03.828002556 +0000 UTC m=+0.127123684 container create 3ff39cf9a6422702885a711f2e5f8db07520bdcf7ced39f058cb472b37b1a8b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=) 2024-09-21T10:09:04.074 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:04 smithi033 podman[132485]: 2024-09-21 10:09:04.07205331 +0000 UTC m=+0.371174430 container init 3ff39cf9a6422702885a711f2e5f8db07520bdcf7ced39f058cb472b37b1a8b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:09:04.497 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:04 smithi033 podman[132485]: 2024-09-21 10:09:04.077930861 +0000 UTC m=+0.377051986 container start 3ff39cf9a6422702885a711f2e5f8db07520bdcf7ced39f058cb472b37b1a8b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-21T10:09:04.497 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:04 smithi033 podman[132485]: 2024-09-21 10:09:04.15514362 +0000 UTC m=+0.454264749 container attach 3ff39cf9a6422702885a711f2e5f8db07520bdcf7ced39f058cb472b37b1a8b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-21T10:09:04.497 INFO:journalctl@ceph.osd.1.smithi033.stdout:Sep 21 10:09:04 smithi033 podman[132485]: 2024-09-21 10:09:04.274361232 +0000 UTC m=+0.573482365 container died 3ff39cf9a6422702885a711f2e5f8db07520bdcf7ced39f058cb472b37b1a8b7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-1-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-21T10:09:04.940 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.1.service' 2024-09-21T10:09:04.981 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:09:04.982 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-21T10:09:04.982 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-21T10:09:04.982 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.2 2024-09-21T10:09:05.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:09:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:09:04.760Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-09-21T10:09:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:09:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:09:04.762Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-09-21T10:09:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:09:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:09:04.762Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-09-21T10:09:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:09:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:09:04.765Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-09-21T10:09:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:09:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:09:04.766Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-09-21T10:09:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:09:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:09:04.766Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-09-21T10:09:05.031 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:09:04 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:09:04.766Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.33:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.33:8765: connect: connection refused" 2024-09-21T10:09:05.246 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:05 smithi033 systemd[1]: Stopping Ceph osd.2 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:09:05.746 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:05 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[113392]: 2024-09-21T10:09:05.329+0000 7fe7b4147640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:09:05.746 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:05 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[113392]: 2024-09-21T10:09:05.329+0000 7fe7b4147640 -1 osd.2 987 *** Got signal Terminated *** 2024-09-21T10:09:05.746 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:05 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2[113392]: 2024-09-21T10:09:05.329+0000 7fe7b4147640 -1 osd.2 987 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:09:07.246 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:06.993Z caller=dispatch.go:353 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:07.246 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:06.994Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:07.246 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:06 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:06.994Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:09:11.148 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:10 smithi033 podman[132613]: 2024-09-21 10:09:10.834545849 +0000 UTC m=+5.622088731 container died 76dbf29565ba59e70f9e81249324f2e546ec2c319fb6e16c10788f09ee8007ff (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, GIT_BRANCH=HEAD, RELEASE=main-62da54b, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-21T10:09:11.496 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:11 smithi033 podman[132613]: 2024-09-21 10:09:11.319847374 +0000 UTC m=+6.107390255 container cleanup 76dbf29565ba59e70f9e81249324f2e546ec2c319fb6e16c10788f09ee8007ff (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-21T10:09:11.496 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:11 smithi033 bash[132613]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2 2024-09-21T10:09:11.907 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:11 smithi033 podman[132633]: 2024-09-21 10:09:11.588071464 +0000 UTC m=+0.755338839 container remove 76dbf29565ba59e70f9e81249324f2e546ec2c319fb6e16c10788f09ee8007ff (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:09:12.234 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:11 smithi033 podman[132729]: 2024-09-21 10:09:11.905466719 +0000 UTC m=+0.104490893 container create f989dd31165a507e53a0dc6644e78eb0c136f94938b5f81204de62a155f2be9b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-62da54b) 2024-09-21T10:09:12.234 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:11 smithi033 podman[132729]: 2024-09-21 10:09:11.817485994 +0000 UTC m=+0.016510138 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:09:12.235 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:12 smithi033 podman[132729]: 2024-09-21 10:09:12.141537289 +0000 UTC m=+0.340561422 container init f989dd31165a507e53a0dc6644e78eb0c136f94938b5f81204de62a155f2be9b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-21T10:09:12.235 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:12 smithi033 podman[132729]: 2024-09-21 10:09:12.14650873 +0000 UTC m=+0.345532865 container start f989dd31165a507e53a0dc6644e78eb0c136f94938b5f81204de62a155f2be9b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-09-21T10:09:12.235 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:12 smithi033 podman[132729]: 2024-09-21 10:09:12.232882767 +0000 UTC m=+0.431906913 container attach f989dd31165a507e53a0dc6644e78eb0c136f94938b5f81204de62a155f2be9b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-21T10:09:12.496 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:12 smithi033 podman[132729]: 2024-09-21 10:09:12.337867008 +0000 UTC m=+0.536891150 container died f989dd31165a507e53a0dc6644e78eb0c136f94938b5f81204de62a155f2be9b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.license=GPLv2, RELEASE=main-62da54b, GIT_CLEAN=True) 2024-09-21T10:09:12.898 INFO:journalctl@ceph.osd.2.smithi033.stdout:Sep 21 10:09:12 smithi033 podman[132772]: 2024-09-21 10:09:12.894464381 +0000 UTC m=+0.550252129 container remove f989dd31165a507e53a0dc6644e78eb0c136f94938b5f81204de62a155f2be9b (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T10:09:12.923 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.2.service' 2024-09-21T10:09:12.966 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:09:12.967 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-21T10:09:12.967 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-21T10:09:12.967 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.3 2024-09-21T10:09:13.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:13 smithi033 systemd[1]: Stopping Ceph osd.3 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:09:13.746 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T10:09:13.298+0000 7f634c45d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:09:13.746 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T10:09:13.298+0000 7f634c45d640 -1 osd.3 987 *** Got signal Terminated *** 2024-09-21T10:09:13.746 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:13 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T10:09:13.298+0000 7f634c45d640 -1 osd.3 987 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:09:15.530 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:15.040+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:15.746 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:15 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T10:09:15.348+0000 7f6348a72640 -1 osd.3 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.039473+0000 front 2024-09-21T10:08:53.039709+0000 (oldest deadline 2024-09-21T10:09:15.339572+0000) 2024-09-21T10:09:16.030 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:16 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:16.003+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:16.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:15 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:15.961+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:16.647 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T10:09:16.352+0000 7f6348a72640 -1 osd.3 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.039473+0000 front 2024-09-21T10:08:53.039709+0000 (oldest deadline 2024-09-21T10:09:15.339572+0000) 2024-09-21T10:09:17.246 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:16.993Z caller=dispatch.go:353 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:09:17.246 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:16.994Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:17.246 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:16 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:16.994Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:09:17.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:17.048+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:17.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:16 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:16.937+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:17.667 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:17 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3[118014]: 2024-09-21T10:09:17.383+0000 7f6348a72640 -1 osd.3 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.039473+0000 front 2024-09-21T10:08:53.039709+0000 (oldest deadline 2024-09-21T10:09:15.339572+0000) 2024-09-21T10:09:17.959 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:17.686+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:18.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:18 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:18.091+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:18.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:17 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:17.957+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:18.947 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:18 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:18.681+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:18.958 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:18 smithi033 podman[132856]: 2024-09-21 10:09:18.704440585 +0000 UTC m=+5.513347832 container died b85b61817088d7f3196a548dee4fa9fc186cdd295c5435d2ed988bd4187c85e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-62da54b, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-21T10:09:19.246 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:19 smithi033 podman[132856]: 2024-09-21 10:09:19.063623778 +0000 UTC m=+5.872531025 container cleanup b85b61817088d7f3196a548dee4fa9fc186cdd295c5435d2ed988bd4187c85e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-21T10:09:19.247 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:19 smithi033 bash[132856]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3 2024-09-21T10:09:19.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:19 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:19.060+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:19.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:18 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:18.945+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:19.281 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:19 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:19.048+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:19.746 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:19 smithi033 podman[132888]: 2024-09-21 10:09:19.431635867 +0000 UTC m=+0.720752690 container remove b85b61817088d7f3196a548dee4fa9fc186cdd295c5435d2ed988bd4187c85e7 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:09:19.912 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:19 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:19.656+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:20.086 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:19 smithi033 podman[132985]: 2024-09-21 10:09:19.734374165 +0000 UTC m=+0.016317447 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:09:20.086 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:19 smithi033 podman[132985]: 2024-09-21 10:09:19.849116765 +0000 UTC m=+0.131060076 container create 6124a4789b5c2d27ee6556988b2ca3d9c255c512addcacbede9dfaad0168eed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-21T10:09:20.087 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:20 smithi033 podman[132985]: 2024-09-21 10:09:20.085001745 +0000 UTC m=+0.366945022 container init 6124a4789b5c2d27ee6556988b2ca3d9c255c512addcacbede9dfaad0168eed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True) 2024-09-21T10:09:20.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:20.088+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:20.281 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:20.072+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:20.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:19 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:19.909+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:20.496 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:20 smithi033 podman[132985]: 2024-09-21 10:09:20.090509313 +0000 UTC m=+0.372452587 container start 6124a4789b5c2d27ee6556988b2ca3d9c255c512addcacbede9dfaad0168eed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, RELEASE=main-62da54b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-21T10:09:20.496 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:20 smithi033 podman[132985]: 2024-09-21 10:09:20.176384264 +0000 UTC m=+0.458327544 container attach 6124a4789b5c2d27ee6556988b2ca3d9c255c512addcacbede9dfaad0168eed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-21T10:09:20.496 INFO:journalctl@ceph.osd.3.smithi033.stdout:Sep 21 10:09:20 smithi033 podman[132985]: 2024-09-21 10:09:20.286313365 +0000 UTC m=+0.568256649 container died 6124a4789b5c2d27ee6556988b2ca3d9c255c512addcacbede9dfaad0168eed6 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-3-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-62da54b) 2024-09-21T10:09:20.875 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:20.614+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:21.005 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.3.service' 2024-09-21T10:09:21.050 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:09:21.050 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-21T10:09:21.050 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-21T10:09:21.050 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.4 2024-09-21T10:09:21.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:21 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:21.058+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:21.281 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:21 smithi105 systemd[1]: Stopping Ceph osd.4 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:09:21.281 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:21 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:21.104+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:21.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:20 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:20.873+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:21.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:21 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:21.589+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:21.780 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:21 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:21.370+0000 7ff7819d1640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:09:21.781 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:21 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:21.370+0000 7ff7819d1640 -1 osd.4 987 *** Got signal Terminated *** 2024-09-21T10:09:21.781 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:21 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:21.370+0000 7ff7819d1640 -1 osd.4 987 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:09:22.121 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:21 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:21.832+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:22.121 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:22.061+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:22.530 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:22.119+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:22.833 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:22.580+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:23.121 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:23.012+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:23.122 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:22 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:22.831+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:23.530 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:23.119+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:23.974 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:23.560+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:23.974 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:23.794+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:23.974 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:23.794+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:24.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:23 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:23.972+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:24.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:24.110+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:24.951 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:24.749+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:24.951 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:24.749+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:24.951 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:24.561+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:24.951 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:24.561+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:25.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:24 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:24.948+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:25.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:25.073+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:25.933 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:25.531+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:25.933 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:25.531+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:25.934 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:25.745+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:25.934 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:25.745+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:26.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:26.045+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:26.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:25.930+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:53.718478+0000 front 2024-09-21T10:08:53.718540+0000 (oldest deadline 2024-09-21T10:09:14.818347+0000) 2024-09-21T10:09:26.281 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:25 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4[92235]: 2024-09-21T10:09:25.930+0000 7ff77dfe6640 -1 osd.4 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.719615+0000 front 2024-09-21T10:09:00.719604+0000 (oldest deadline 2024-09-21T10:09:25.419402+0000) 2024-09-21T10:09:26.749 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:26.489+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:26.750 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:26.489+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:27.030 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:26 smithi105 podman[119332]: 2024-09-21 10:09:26.925693863 +0000 UTC m=+5.678614581 container died 01f4f66dbe7202ac8254b91223c9bbf298c172f91d3166bdeab7eff23ecbbea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b) 2024-09-21T10:09:27.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:26.747+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:27.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:26 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:26.747+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:27.345 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:27.051+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:27.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:26 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:26.994Z caller=dispatch.go:353 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:27.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:26 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:26.995Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:27.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:26 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:26.995Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:09:27.597 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:27 smithi105 podman[119332]: 2024-09-21 10:09:27.343993854 +0000 UTC m=+6.096914583 container cleanup 01f4f66dbe7202ac8254b91223c9bbf298c172f91d3166bdeab7eff23ecbbea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=) 2024-09-21T10:09:27.597 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:27 smithi105 bash[119332]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4 2024-09-21T10:09:27.597 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:27.447+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:27.598 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:27.447+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:28.006 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:27 smithi105 podman[119352]: 2024-09-21 10:09:27.595319977 +0000 UTC m=+0.661285958 container remove 01f4f66dbe7202ac8254b91223c9bbf298c172f91d3166bdeab7eff23ecbbea1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2) 2024-09-21T10:09:28.006 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:27.745+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:28.006 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:27 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:27.745+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:28.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:28.083+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:28.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:28.083+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:02.153065+0000 front 2024-09-21T10:09:02.153053+0000 (oldest deadline 2024-09-21T10:09:28.052947+0000) 2024-09-21T10:09:28.280 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:28 smithi105 podman[119447]: 2024-09-21 10:09:28.00454913 +0000 UTC m=+0.109894589 container create 17c714d91b38669df3a75186840634b9ca879fb3d3142bdf021927511a87c91d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-09-21T10:09:28.281 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:28 smithi105 podman[119447]: 2024-09-21 10:09:27.910942328 +0000 UTC m=+0.016287802 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:09:28.281 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:28 smithi105 podman[119447]: 2024-09-21 10:09:28.215264744 +0000 UTC m=+0.320610203 container init 17c714d91b38669df3a75186840634b9ca879fb3d3142bdf021927511a87c91d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731) 2024-09-21T10:09:28.281 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:28 smithi105 podman[119447]: 2024-09-21 10:09:28.220375944 +0000 UTC m=+0.325721404 container start 17c714d91b38669df3a75186840634b9ca879fb3d3142bdf021927511a87c91d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=main-62da54b) 2024-09-21T10:09:28.738 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:28.453+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:28.738 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:28.453+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:28.738 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:28 smithi105 podman[119447]: 2024-09-21 10:09:28.290059985 +0000 UTC m=+0.395405459 container attach 17c714d91b38669df3a75186840634b9ca879fb3d3142bdf021927511a87c91d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:09:28.739 INFO:journalctl@ceph.osd.4.smithi105.stdout:Sep 21 10:09:28 smithi105 podman[119447]: 2024-09-21 10:09:28.419198983 +0000 UTC m=+0.524544455 container died 17c714d91b38669df3a75186840634b9ca879fb3d3142bdf021927511a87c91d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-4-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-21T10:09:29.028 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:28.736+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:29.029 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:28 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:28.736+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:29.052 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.4.service' 2024-09-21T10:09:29.091 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:09:29.091 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-21T10:09:29.091 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-21T10:09:29.092 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.5 2024-09-21T10:09:29.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:29.115+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:29.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:29.115+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:02.153065+0000 front 2024-09-21T10:09:02.153053+0000 (oldest deadline 2024-09-21T10:09:28.052947+0000) 2024-09-21T10:09:29.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:29 smithi105 systemd[1]: Stopping Ceph osd.5 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:09:29.759 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:29.446+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:29.760 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:29.446+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:29.760 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:29.415+0000 7f42d6691640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:09:29.760 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:29.415+0000 7f42d6691640 -1 osd.5 987 *** Got signal Terminated *** 2024-09-21T10:09:29.760 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:29.415+0000 7f42d6691640 -1 osd.5 987 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:09:30.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:29.757+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:30.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:29 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:29.757+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:30.407 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:30.152+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:30.408 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:30.152+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:02.153065+0000 front 2024-09-21T10:09:02.153053+0000 (oldest deadline 2024-09-21T10:09:28.052947+0000) 2024-09-21T10:09:30.723 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:30.405+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:30.723 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:30.405+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:31.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:30.721+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:31.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:30 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:30.721+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:31.530 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:31.199+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:31.530 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:31.199+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:02.153065+0000 front 2024-09-21T10:09:02.153053+0000 (oldest deadline 2024-09-21T10:09:28.052947+0000) 2024-09-21T10:09:31.531 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:31.415+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:31.531 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:31.415+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:31.531 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:31.415+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:32.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:31.746+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:32.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:31 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:31.746+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:32.457 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:32.196+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:32.458 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:32.196+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:02.153065+0000 front 2024-09-21T10:09:02.153053+0000 (oldest deadline 2024-09-21T10:09:28.052947+0000) 2024-09-21T10:09:32.458 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:32.196+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:08.553661+0000 front 2024-09-21T10:09:08.553749+0000 (oldest deadline 2024-09-21T10:09:31.453748+0000) 2024-09-21T10:09:32.752 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:32.455+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:32.753 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:32.455+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:32.753 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:32.455+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:33.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:32.750+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:33.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:32.750+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:33.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:32 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:32.750+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:33.478 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:33.173+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:33.479 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:33.173+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:02.153065+0000 front 2024-09-21T10:09:02.153053+0000 (oldest deadline 2024-09-21T10:09:28.052947+0000) 2024-09-21T10:09:33.479 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:33.173+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:08.553661+0000 front 2024-09-21T10:09:08.553749+0000 (oldest deadline 2024-09-21T10:09:31.453748+0000) 2024-09-21T10:09:33.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:33.476+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:33.781 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:33.476+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:33.781 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:33.476+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:33.782 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:33.715+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:33.782 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:33.715+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:33.782 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:33 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:33.715+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:34.460 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:34.163+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:52.251682+0000 front 2024-09-21T10:08:52.251742+0000 (oldest deadline 2024-09-21T10:09:18.151529+0000) 2024-09-21T10:09:34.514 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:34.163+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:02.153065+0000 front 2024-09-21T10:09:02.153053+0000 (oldest deadline 2024-09-21T10:09:28.052947+0000) 2024-09-21T10:09:34.514 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5[96938]: 2024-09-21T10:09:34.163+0000 7f42d2ca6640 -1 osd.5 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:08.553661+0000 front 2024-09-21T10:09:08.553749+0000 (oldest deadline 2024-09-21T10:09:31.453748+0000) 2024-09-21T10:09:34.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:34.458+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:34.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:34.458+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:34.780 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:34.458+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:34.781 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:34.671+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:34.781 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:34.671+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:34.781 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:34 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:34.671+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:35.280 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:35 smithi105 podman[119576]: 2024-09-21 10:09:35.013814204 +0000 UTC m=+5.702226635 container died 4ff1257a8be148977636a39842a9a0a3a38e2b333dbe1223f8d1fc4403c623c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-62da54b, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731) 2024-09-21T10:09:35.607 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:35 smithi105 podman[119576]: 2024-09-21 10:09:35.396494666 +0000 UTC m=+6.084907081 container cleanup 4ff1257a8be148977636a39842a9a0a3a38e2b333dbe1223f8d1fc4403c623c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-21T10:09:35.608 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:35 smithi105 bash[119576]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5 2024-09-21T10:09:35.608 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:35.414+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:35.608 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:35.414+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:35.608 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:35.414+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:35.930 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:35 smithi105 podman[119600]: 2024-09-21 10:09:35.605952273 +0000 UTC m=+0.594439910 container remove 4ff1257a8be148977636a39842a9a0a3a38e2b333dbe1223f8d1fc4403c623c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5, RELEASE=main-62da54b, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-09-21T10:09:35.930 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:35.707+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:35.930 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:35.707+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:35.930 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:35 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:35.707+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:36.261 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:35 smithi105 podman[119690]: 2024-09-21 10:09:35.831188143 +0000 UTC m=+0.015548416 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:09:36.261 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:35 smithi105 podman[119690]: 2024-09-21 10:09:35.948740672 +0000 UTC m=+0.133100932 container create 96d061042c250d08dc399d0dcc7e4ca16b0c5f40c1288d2293a99a058088ebfe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2) 2024-09-21T10:09:36.261 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:36 smithi105 podman[119690]: 2024-09-21 10:09:36.168115723 +0000 UTC m=+0.352475995 container init 96d061042c250d08dc399d0dcc7e4ca16b0c5f40c1288d2293a99a058088ebfe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-21T10:09:36.261 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:36 smithi105 podman[119690]: 2024-09-21 10:09:36.17302 +0000 UTC m=+0.357380276 container start 96d061042c250d08dc399d0dcc7e4ca16b0c5f40c1288d2293a99a058088ebfe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-62da54b) 2024-09-21T10:09:36.261 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:36 smithi105 podman[119690]: 2024-09-21 10:09:36.259479023 +0000 UTC m=+0.443839303 container attach 96d061042c250d08dc399d0dcc7e4ca16b0c5f40c1288d2293a99a058088ebfe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0) 2024-09-21T10:09:36.530 INFO:journalctl@ceph.osd.5.smithi105.stdout:Sep 21 10:09:36 smithi105 podman[119690]: 2024-09-21 10:09:36.378267397 +0000 UTC m=+0.562627665 container died 96d061042c250d08dc399d0dcc7e4ca16b0c5f40c1288d2293a99a058088ebfe (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-5-deactivate, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-21T10:09:36.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:36.412+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:36.531 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:36.412+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:36.531 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:36.412+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:36.956 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:36.694+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:36.956 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:36.694+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:36.957 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:36 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:36.694+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:36.987 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.5.service' 2024-09-21T10:09:37.026 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:09:37.027 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-21T10:09:37.027 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-21T10:09:37.027 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.6 2024-09-21T10:09:37.280 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:37 smithi105 systemd[1]: Stopping Ceph osd.6 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:09:37.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:36 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:36.996Z caller=dispatch.go:353 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:09:37.496 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:36 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:36.996Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:37.497 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:36 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:36.997Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:09:37.729 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:37.409+0000 7efdc2e04640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:09:37.729 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:37.409+0000 7efdc2e04640 -1 osd.6 987 *** Got signal Terminated *** 2024-09-21T10:09:37.730 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:37.409+0000 7efdc2e04640 -1 osd.6 987 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:09:37.730 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:37.412+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:37.730 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:37.412+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:37.730 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:37.412+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:38.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:37.726+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:38.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:37.726+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:38.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:37 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:37.726+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:38.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:38 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:38.426+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:38.530 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:38 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:38.426+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:38.531 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:38 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:38.426+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:39.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:38 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:38.678+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:39.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:38 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:38.678+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:39.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:38 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:38.678+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:39.728 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:39 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:39.389+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:39.728 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:39 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:39.389+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:39.728 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:39 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:39.389+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:39.728 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:39 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:39.389+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:18.169517+0000 front 2024-09-21T10:09:18.169497+0000 (oldest deadline 2024-09-21T10:09:39.269429+0000) 2024-09-21T10:09:40.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:39 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:39.726+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:40.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:39 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:39.726+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:40.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:39 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:39.726+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:40.698 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:40.358+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:40.698 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:40.358+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:40.698 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:40.358+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:40.699 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:40.358+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:18.169517+0000 front 2024-09-21T10:09:18.169497+0000 (oldest deadline 2024-09-21T10:09:39.269429+0000) 2024-09-21T10:09:41.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:40.696+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:41.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:40.696+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:41.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:40 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:40.696+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:41.690 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:41.392+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:41.690 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:41.392+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:41.690 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:41.392+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:41.690 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:41.392+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:18.169517+0000 front 2024-09-21T10:09:18.169497+0000 (oldest deadline 2024-09-21T10:09:39.269429+0000) 2024-09-21T10:09:42.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:41.686+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:42.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:41.686+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:42.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:41 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:41.686+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:42.733 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:42.345+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:51.466166+0000 front 2024-09-21T10:08:51.466158+0000 (oldest deadline 2024-09-21T10:09:16.766145+0000) 2024-09-21T10:09:42.734 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:42.345+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:01.967587+0000 front 2024-09-21T10:09:01.967507+0000 (oldest deadline 2024-09-21T10:09:24.267272+0000) 2024-09-21T10:09:42.734 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:42.345+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:10.168221+0000 front 2024-09-21T10:09:10.168171+0000 (oldest deadline 2024-09-21T10:09:31.168290+0000) 2024-09-21T10:09:42.734 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6[101759]: 2024-09-21T10:09:42.345+0000 7efdbb411640 -1 osd.6 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:18.169517+0000 front 2024-09-21T10:09:18.169497+0000 (oldest deadline 2024-09-21T10:09:39.269429+0000) 2024-09-21T10:09:43.030 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:42 smithi105 podman[119819]: 2024-09-21 10:09:42.760563141 +0000 UTC m=+5.471665994 container died 132bc834f24d6afd074a39b8c8791c6833709b42389d6c9e312aaca69f98941a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T10:09:43.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:42.730+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:43.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:42.730+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:43.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:42.730+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:43.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:42 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:42.730+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:17.016350+0000 front 2024-09-21T10:09:17.016365+0000 (oldest deadline 2024-09-21T10:09:41.716120+0000) 2024-09-21T10:09:43.410 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:43 smithi105 podman[119819]: 2024-09-21 10:09:43.132757903 +0000 UTC m=+5.843860758 container cleanup 132bc834f24d6afd074a39b8c8791c6833709b42389d6c9e312aaca69f98941a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-62da54b) 2024-09-21T10:09:43.410 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:43 smithi105 bash[119819]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6 2024-09-21T10:09:43.755 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:43 smithi105 podman[119852]: 2024-09-21 10:09:43.408983943 +0000 UTC m=+0.649215770 container remove 132bc834f24d6afd074a39b8c8791c6833709b42389d6c9e312aaca69f98941a (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, RELEASE=main-62da54b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:09:44.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:43 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:43.759+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:44.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:43 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:43.759+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:44.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:43 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:43.759+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:44.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:43 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:43.759+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:17.016350+0000 front 2024-09-21T10:09:17.016365+0000 (oldest deadline 2024-09-21T10:09:41.716120+0000) 2024-09-21T10:09:44.031 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:43 smithi105 podman[119948]: 2024-09-21 10:09:43.656243085 +0000 UTC m=+0.015563684 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:09:44.031 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:43 smithi105 podman[119948]: 2024-09-21 10:09:43.76812128 +0000 UTC m=+0.127441920 container create 4a27942270d77990c84903a4600fa60b64fcd405d9effcb15d606f043dbe0f9c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T10:09:44.031 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:43 smithi105 podman[119948]: 2024-09-21 10:09:43.995541185 +0000 UTC m=+0.354861785 container init 4a27942270d77990c84903a4600fa60b64fcd405d9effcb15d606f043dbe0f9c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-09-21T10:09:44.031 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:44 smithi105 podman[119948]: 2024-09-21 10:09:44.000536098 +0000 UTC m=+0.359856704 container start 4a27942270d77990c84903a4600fa60b64fcd405d9effcb15d606f043dbe0f9c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:09:44.467 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:44 smithi105 podman[119948]: 2024-09-21 10:09:44.095640658 +0000 UTC m=+0.454961263 container attach 4a27942270d77990c84903a4600fa60b64fcd405d9effcb15d606f043dbe0f9c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=main-62da54b, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True) 2024-09-21T10:09:44.467 INFO:journalctl@ceph.osd.6.smithi105.stdout:Sep 21 10:09:44 smithi105 podman[119948]: 2024-09-21 10:09:44.189561107 +0000 UTC m=+0.548881703 container died 4a27942270d77990c84903a4600fa60b64fcd405d9effcb15d606f043dbe0f9c (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-21T10:09:44.790 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.6.service' 2024-09-21T10:09:44.829 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:09:44.829 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-21T10:09:44.829 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-21T10:09:44.829 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.7 2024-09-21T10:09:45.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:44.796+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:45.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:44.796+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:45.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:44.796+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:45.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:44 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:44.796+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:17.016350+0000 front 2024-09-21T10:09:17.016365+0000 (oldest deadline 2024-09-21T10:09:41.716120+0000) 2024-09-21T10:09:45.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:44 smithi105 systemd[1]: Stopping Ceph osd.7 for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:09:45.530 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:45.161+0000 7fc1abced640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:09:45.531 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:45.161+0000 7fc1abced640 -1 osd.7 987 *** Got signal Terminated *** 2024-09-21T10:09:45.531 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:45.161+0000 7fc1abced640 -1 osd.7 987 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-21T10:09:46.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:45.843+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:46.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:45.843+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:46.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:45.843+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:46.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:45 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:45.843+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:17.016350+0000 front 2024-09-21T10:09:17.016365+0000 (oldest deadline 2024-09-21T10:09:41.716120+0000) 2024-09-21T10:09:47.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:46.857+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:47.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:46.857+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:47.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:46.857+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:47.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:46 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:46.857+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:17.016350+0000 front 2024-09-21T10:09:17.016365+0000 (oldest deadline 2024-09-21T10:09:41.716120+0000) 2024-09-21T10:09:47.408 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:46.996Z caller=dispatch.go:353 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:47.408 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:46.997Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:47.408 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:46 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:46.997Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:09:48.030 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:47.837+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:48.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:47.837+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:48.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:47.837+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:48.031 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:47 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:47.837+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:17.016350+0000 front 2024-09-21T10:09:17.016365+0000 (oldest deadline 2024-09-21T10:09:41.716120+0000) 2024-09-21T10:09:49.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:48.798+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:49.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:48.798+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:49.280 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:48.798+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:49.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:48 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:48.798+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:17.016350+0000 front 2024-09-21T10:09:17.016365+0000 (oldest deadline 2024-09-21T10:09:41.716120+0000) 2024-09-21T10:09:50.282 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:49.807+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6806 osd.0 since back 2024-09-21T10:08:49.112485+0000 front 2024-09-21T10:08:49.112637+0000 (oldest deadline 2024-09-21T10:09:15.012508+0000) 2024-09-21T10:09:50.282 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:49.807+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6814 osd.1 since back 2024-09-21T10:09:00.813898+0000 front 2024-09-21T10:09:00.813838+0000 (oldest deadline 2024-09-21T10:09:23.713828+0000) 2024-09-21T10:09:50.282 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:49.807+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6822 osd.2 since back 2024-09-21T10:09:07.115079+0000 front 2024-09-21T10:09:07.114960+0000 (oldest deadline 2024-09-21T10:09:32.414843+0000) 2024-09-21T10:09:50.282 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:49 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7[106579]: 2024-09-21T10:09:49.807+0000 7fc1a42fa640 -1 osd.7 987 heartbeat_check: no reply from 172.21.15.33:6830 osd.3 since back 2024-09-21T10:09:17.016350+0000 front 2024-09-21T10:09:17.016365+0000 (oldest deadline 2024-09-21T10:09:41.716120+0000) 2024-09-21T10:09:50.905 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:50 smithi105 podman[120075]: 2024-09-21 10:09:50.627842481 +0000 UTC m=+5.575224359 container died ea10ccd8e9ee9580ab0c729f3addb87e59e1f7e048bff1bd4567b9602cbac03d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, GIT_BRANCH=HEAD, RELEASE=main-62da54b, GIT_CLEAN=True, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2) 2024-09-21T10:09:51.271 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:51 smithi105 podman[120075]: 2024-09-21 10:09:51.001838011 +0000 UTC m=+5.949219931 container cleanup ea10ccd8e9ee9580ab0c729f3addb87e59e1f7e048bff1bd4567b9602cbac03d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:09:51.271 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:51 smithi105 bash[120075]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7 2024-09-21T10:09:51.530 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:51 smithi105 podman[120095]: 2024-09-21 10:09:51.269733842 +0000 UTC m=+0.639209643 container remove ea10ccd8e9ee9580ab0c729f3addb87e59e1f7e048bff1bd4567b9602cbac03d (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, RELEASE=main-62da54b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-21T10:09:51.882 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:51 smithi105 podman[120191]: 2024-09-21 10:09:51.515445162 +0000 UTC m=+0.015559776 image pull a8ebb3d62dd1a69d83c65ce6bcc14c50b43d220d13e8424b0cbe758f609711f0 quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47 2024-09-21T10:09:51.883 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:51 smithi105 podman[120191]: 2024-09-21 10:09:51.645576712 +0000 UTC m=+0.145691327 container create 7ed55490c2a7c8980feceeba737e68bd3d4f85a6044c6c10ca2309a947b6aba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-62da54b, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-21T10:09:51.883 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:51 smithi105 podman[120191]: 2024-09-21 10:09:51.881078759 +0000 UTC m=+0.381193381 container init 7ed55490c2a7c8980feceeba737e68bd3d4f85a6044c6c10ca2309a947b6aba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:09:52.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:51 smithi105 podman[120191]: 2024-09-21 10:09:51.886190712 +0000 UTC m=+0.386305327 container start 7ed55490c2a7c8980feceeba737e68bd3d4f85a6044c6c10ca2309a947b6aba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-62da54b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-21T10:09:52.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:51 smithi105 podman[120191]: 2024-09-21 10:09:51.983290532 +0000 UTC m=+0.483405157 container attach 7ed55490c2a7c8980feceeba737e68bd3d4f85a6044c6c10ca2309a947b6aba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, RELEASE=main-62da54b, org.label-schema.schema-version=1.0) 2024-09-21T10:09:52.281 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:52 smithi105 podman[120191]: 2024-09-21 10:09:52.079774888 +0000 UTC m=+0.579889516 container died 7ed55490c2a7c8980feceeba737e68bd3d4f85a6044c6c10ca2309a947b6aba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, RELEASE=main-62da54b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23) 2024-09-21T10:09:52.675 INFO:journalctl@ceph.osd.7.smithi105.stdout:Sep 21 10:09:52 smithi105 podman[120234]: 2024-09-21 10:09:52.661431091 +0000 UTC m=+0.576394449 container remove 7ed55490c2a7c8980feceeba737e68bd3d4f85a6044c6c10ca2309a947b6aba5 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-osd-7-deactivate, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-62da54b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-09-21T10:09:52.682 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@osd.7.service' 2024-09-21T10:09:52.724 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:09:52.724 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-21T10:09:52.724 INFO:tasks.cephadm.mds.a:Stopping mds.a... 2024-09-21T10:09:52.725 DEBUG:teuthology.orchestra.run.smithi033:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.a 2024-09-21T10:09:53.150 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:09:52 smithi033 systemd[1]: Stopping Ceph mds.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:09:53.496 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:09:53 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[123139]: 2024-09-21T10:09:53.147+0000 7f7bc6ed6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:09:53.496 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:09:53 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a[123139]: 2024-09-21T10:09:53.147+0000 7f7bc6ed6640 -1 mds.a *** got signal Terminated *** 2024-09-21T10:09:56.982 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:09:56 smithi033 podman[133127]: 2024-09-21 10:09:56.625539123 +0000 UTC m=+3.700575068 container died 9bc95e6ef3d05d51eaccf33b583cd4fba918901e61a472cb1f121845798f2645 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240731, RELEASE=main-62da54b) 2024-09-21T10:09:57.246 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:09:57 smithi033 podman[133127]: 2024-09-21 10:09:57.069573299 +0000 UTC m=+4.144609234 container cleanup 9bc95e6ef3d05d51eaccf33b583cd4fba918901e61a472cb1f121845798f2645 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-21T10:09:57.246 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:09:57 smithi033 bash[133127]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a 2024-09-21T10:09:57.247 INFO:journalctl@ceph.mds.a.smithi033.stdout:Sep 21 10:09:57 smithi033 podman[133144]: 2024-09-21 10:09:57.220572002 +0000 UTC m=+0.590268811 container remove 9bc95e6ef3d05d51eaccf33b583cd4fba918901e61a472cb1f121845798f2645 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-a, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, RELEASE=main-62da54b, io.buildah.version=1.37.2, GIT_CLEAN=True) 2024-09-21T10:09:57.247 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:56 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:56.998Z caller=dispatch.go:353 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:57.247 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:56 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:56.998Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:09:57.247 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:09:56 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:09:56.999Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephNodeDiskspaceWarning\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:09:57.465 DEBUG:teuthology.orchestra.run.smithi033:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.a.service' 2024-09-21T10:09:57.501 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:09:57.501 INFO:tasks.cephadm.mds.a:Stopped mds.a 2024-09-21T10:09:57.502 INFO:tasks.cephadm.mds.b:Stopping mds.b... 2024-09-21T10:09:57.502 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.b 2024-09-21T10:09:57.780 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:09:57 smithi105 systemd[1]: Stopping Ceph mds.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:09:58.282 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:09:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b[111156]: 2024-09-21T10:09:57.824+0000 7fd2d5286640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mds -n mds.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-21T10:09:58.282 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:09:57 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b[111156]: 2024-09-21T10:09:57.824+0000 7fd2d5286640 -1 mds.b *** got signal Terminated *** 2024-09-21T10:10:02.108 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:10:01 smithi105 podman[120318]: 2024-09-21 10:10:01.835625812 +0000 UTC m=+4.136132783 container died 1ceb42a1686b3fa095312abf75740c3f302bc7bc8e3eee755c5960d89e693fd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-21T10:10:02.484 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:10:02 smithi105 podman[120318]: 2024-09-21 10:10:02.205414299 +0000 UTC m=+4.505921269 container cleanup 1ceb42a1686b3fa095312abf75740c3f302bc7bc8e3eee755c5960d89e693fd1 (image=quay.ceph.io/ceph-ci/ceph@sha256:1283e169fa2a012678b44d89aba6a8833eba9b66e43c608c048a2ff9c690cf47, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-62da54b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=92e6e0d02ffae0030ac4c9454206214b98a73a23, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-21T10:10:02.484 INFO:journalctl@ceph.mds.b.smithi105.stdout:Sep 21 10:10:02 smithi105 bash[120318]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-mds-b 2024-09-21T10:10:02.626 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@mds.b.service' 2024-09-21T10:10:02.662 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:10:02.662 INFO:tasks.cephadm.mds.b:Stopped mds.b 2024-09-21T10:10:02.663 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-21T10:10:02.663 DEBUG:teuthology.orchestra.run.smithi105:> sudo systemctl stop ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a 2024-09-21T10:10:03.030 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:02 smithi105 systemd[1]: Stopping Ceph prometheus.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:10:03.421 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.049Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-21T10:10:03.421 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.049Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-21T10:10:03.421 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.049Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-21T10:10:03.421 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.049Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-21T10:10:03.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.049Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-21T10:10:03.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.049Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-21T10:10:03.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.049Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-21T10:10:03.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.049Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-21T10:10:03.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.050Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-21T10:10:03.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.051Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-21T10:10:03.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.051Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-21T10:10:03.422 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a[78759]: ts=2024-09-21T10:10:03.051Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-21T10:10:03.423 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 podman[120452]: 2024-09-21 10:10:03.060431902 +0000 UTC m=+0.173276156 container died 670cdb3edd9d943943f96657260d47591a5d85802bcd87a16e847142b4dd80fe (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T10:10:03.746 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:03 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:10:03.248Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] aggrGroup="{}/{}:{alertname=\"CephMgrPrometheusModuleInactive\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.33:8443: connect: connection refused" 2024-09-21T10:10:03.746 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:03 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:10:03.248Z caller=notify.go:848 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] aggrGroup="{}/{}:{alertname=\"CephMgrPrometheusModuleInactive\"}" msg="Notify attempt failed, will retry later" attempts=1 err="Post \"\": dial tcp 172.21.15.105:8443: connect: connection refused" 2024-09-21T10:10:03.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 podman[120452]: 2024-09-21 10:10:03.534263362 +0000 UTC m=+0.647107605 container cleanup 670cdb3edd9d943943f96657260d47591a5d85802bcd87a16e847142b4dd80fe (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-21T10:10:03.778 INFO:journalctl@ceph.prometheus.a.smithi105.stdout:Sep 21 10:10:03 smithi105 bash[120452]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-prometheus-a 2024-09-21T10:10:03.907 DEBUG:teuthology.orchestra.run.smithi105:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a29e8460-77fd-11ef-baf6-efdc52797490@prometheus.a.service' 2024-09-21T10:10:03.939 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-21T10:10:03.939 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-21T10:10:03.939 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a29e8460-77fd-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:10:05.928 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:05 smithi033 systemd[1]: Stopping Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:10:06.246 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:05 smithi033 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a[87564]: ts=2024-09-21T10:10:05.926Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-21T10:10:06.246 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:06 smithi033 podman[133580]: 2024-09-21 10:10:06.090592932 +0000 UTC m=+0.273567518 container died 9ab68da7db7a9e5850b99ad6b2710c271f6b9ce06c15a22cacca4dae83a50e20 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:10:06.660 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:06 smithi033 podman[133580]: 2024-09-21 10:10:06.424144932 +0000 UTC m=+0.607119505 container cleanup 9ab68da7db7a9e5850b99ad6b2710c271f6b9ce06c15a22cacca4dae83a50e20 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:10:06.660 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:06 smithi033 bash[133580]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a 2024-09-21T10:10:06.661 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:06 smithi033 podman[133596]: 2024-09-21 10:10:06.583700335 +0000 UTC m=+0.492757118 container remove 9ab68da7db7a9e5850b99ad6b2710c271f6b9ce06c15a22cacca4dae83a50e20 (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-21T10:10:06.661 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:06 smithi033 podman[133596]: 2024-09-21 10:10:06.659201716 +0000 UTC m=+0.568258454 volume remove 9dbf7a68c2ae6b2754434ca98eb7844214a3b83f98fb78c472ffa547d01874ed 2024-09-21T10:10:06.924 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:06 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Deactivated successfully. 2024-09-21T10:10:06.924 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:06 smithi033 systemd[1]: Stopped Ceph alertmanager.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:10:06.924 INFO:journalctl@ceph.alertmanager.a.smithi033.stdout:Sep 21 10:10:06 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@alertmanager.a.service: Consumed 1.924s CPU time. 2024-09-21T10:10:07.246 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:07 smithi033 systemd[1]: Stopping Ceph node-exporter.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:10:07.746 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:07 smithi033 podman[133713]: 2024-09-21 10:10:07.441765349 +0000 UTC m=+0.211502733 container died cce524477a32bf23f87c2f1e136097a4865637dde62b79b35bc39e45ea32f0df (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:10:08.215 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:07 smithi033 podman[133713]: 2024-09-21 10:10:07.878411137 +0000 UTC m=+0.648148516 container cleanup cce524477a32bf23f87c2f1e136097a4865637dde62b79b35bc39e45ea32f0df (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:10:08.216 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:07 smithi033 bash[133713]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a 2024-09-21T10:10:08.216 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:08 smithi033 podman[133730]: 2024-09-21 10:10:08.02116889 +0000 UTC m=+0.583062709 container remove cce524477a32bf23f87c2f1e136097a4865637dde62b79b35bc39e45ea32f0df (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-21T10:10:08.216 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:08 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:10:08.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:08 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-21T10:10:08.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:08 smithi033 systemd[1]: Stopped Ceph node-exporter.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:10:08.496 INFO:journalctl@ceph.node-exporter.a.smithi033.stdout:Sep 21 10:10:08 smithi033 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.a.service: Consumed 3.784s CPU time. 2024-09-21T10:10:08.918 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a29e8460-77fd-11ef-baf6-efdc52797490 --force --keep-logs 2024-09-21T10:10:10.944 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:10 smithi105 systemd[1]: Stopping Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:10:11.234 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=server t=2024-09-21T10:10:10.942105811Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-21T10:10:11.235 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=ticker t=2024-09-21T10:10:10.942529951Z level=info msg=stopped last_tick=2024-09-21T10:10:10Z 2024-09-21T10:10:11.235 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=tracing t=2024-09-21T10:10:10.942545395Z level=info msg="Closing tracing" 2024-09-21T10:10:11.235 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=grafana-apiserver t=2024-09-21T10:10:10.94256047Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-21T10:10:11.235 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:10 smithi105 ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a[115162]: logger=sqlstore.transactions t=2024-09-21T10:10:10.954007544Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-21T10:10:11.235 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:10 smithi105 podman[120923]: 2024-09-21 10:10:10.962809445 +0000 UTC m=+0.122735451 container died 37486b9e0f8e6daa0783f1864670eaec0c98bbd14558ca52bcc45722d31328e1 (image=quay.io/ceph/grafana:10.4.8, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:10:11.530 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:11 smithi105 podman[120923]: 2024-09-21 10:10:11.322749345 +0000 UTC m=+0.482675343 container cleanup 37486b9e0f8e6daa0783f1864670eaec0c98bbd14558ca52bcc45722d31328e1 (image=quay.io/ceph/grafana:10.4.8, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:10:11.530 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:11 smithi105 bash[120923]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a 2024-09-21T10:10:11.930 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:11 smithi105 podman[120940]: 2024-09-21 10:10:11.590810553 +0000 UTC m=+0.627256369 container remove 37486b9e0f8e6daa0783f1864670eaec0c98bbd14558ca52bcc45722d31328e1 (image=quay.io/ceph/grafana:10.4.8, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-grafana-a, maintainer=Grafana Labs ) 2024-09-21T10:10:11.931 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:11 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service: Deactivated successfully. 2024-09-21T10:10:11.931 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:11 smithi105 systemd[1]: Stopped Ceph grafana.a for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:10:11.931 INFO:journalctl@ceph.grafana.a.smithi105.stdout:Sep 21 10:10:11 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@grafana.a.service: Consumed 7.000s CPU time. 2024-09-21T10:10:12.280 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:11 smithi105 systemd[1]: Stopping Ceph node-exporter.b for a29e8460-77fd-11ef-baf6-efdc52797490... 2024-09-21T10:10:12.280 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:12 smithi105 podman[121056]: 2024-09-21 10:10:12.17300836 +0000 UTC m=+0.105506875 container died c79287d80715ac184038d94d33f856407597b1b8d3c82429cb0c0b2cc20f0e18 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:10:12.780 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:12 smithi105 podman[121056]: 2024-09-21 10:10:12.443234388 +0000 UTC m=+0.375732944 container cleanup c79287d80715ac184038d94d33f856407597b1b8d3c82429cb0c0b2cc20f0e18 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:10:12.780 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:12 smithi105 bash[121056]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b 2024-09-21T10:10:12.780 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:12 smithi105 podman[121072]: 2024-09-21 10:10:12.602551263 +0000 UTC m=+0.430916336 container remove c79287d80715ac184038d94d33f856407597b1b8d3c82429cb0c0b2cc20f0e18 (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-a29e8460-77fd-11ef-baf6-efdc52797490-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-21T10:10:12.780 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:12 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-21T10:10:13.227 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:12 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-21T10:10:13.227 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:12 smithi105 systemd[1]: Stopped Ceph node-exporter.b for a29e8460-77fd-11ef-baf6-efdc52797490. 2024-09-21T10:10:13.227 INFO:journalctl@ceph.node-exporter.b.smithi105.stdout:Sep 21 10:10:12 smithi105 systemd[1]: ceph-a29e8460-77fd-11ef-baf6-efdc52797490@node-exporter.b.service: Consumed 3.494s CPU time. 2024-09-21T10:10:13.445 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:10:13.480 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-21T10:10:13.507 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-21T10:10:13.509 DEBUG:teuthology.misc:Transferring archived files from smithi033:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658/remote/smithi033/crash 2024-09-21T10:10:13.510 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:10:13.570 INFO:teuthology.orchestra.run.smithi033.stderr:tar: /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:10:13.570 INFO:teuthology.orchestra.run.smithi033.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:10:13.572 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/crash to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658/remote/smithi105/crash 2024-09-21T10:10:13.573 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/crash -- . 2024-09-21T10:10:13.606 INFO:teuthology.orchestra.run.smithi105.stderr:tar: /var/lib/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/crash: Cannot open: No such file or directory 2024-09-21T10:10:13.606 INFO:teuthology.orchestra.run.smithi105.stderr:tar: Error is not recoverable: exiting now 2024-09-21T10:10:13.608 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-21T10:10:13.608 DEBUG:teuthology.orchestra.run.smithi033:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(POOL_APP_NOT_ENABLED\)' | egrep -v OSD_DOWN | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_DEGRADED | egrep -v 'Reduced data availability' | egrep -v 'Degraded data redundancy' | egrep -v OSDMAP_FLAGS | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-21T10:10:13.642 INFO:teuthology.orchestra.run.smithi033.stdout:2024-09-21T09:59:37.504862+0000 mon.a (mon.0) 820 : cluster [WRN] Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-09-21T10:10:13.642 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-21T10:10:13.642 INFO:tasks.cephadm:Compressing logs... 2024-09-21T10:10:13.643 DEBUG:teuthology.orchestra.run.smithi033:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:10:13.685 DEBUG:teuthology.orchestra.run.smithi105:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-21T10:10:13.709 INFO:teuthology.orchestra.run.smithi105.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.7.log 2024-09-21T10:10:13.709 INFO:teuthology.orchestra.run.smithi105.stderr:: No such file or directory 2024-09-21T10:10:13.709 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:10:13.710 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mds.b.log 2024-09-21T10:10:13.711 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.4.log 2024-09-21T10:10:13.711 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mds.b.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:10:13.711 INFO:teuthology.orchestra.run.smithi105.stderr: 84.0% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mds.b.log.gz 2024-09-21T10:10:13.711 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:10:13.713 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.4.log: /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.6.log 2024-09-21T10:10:13.713 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.5.log 2024-09-21T10:10:13.714 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.6.log: 84.1% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:10:13.715 INFO:teuthology.orchestra.run.smithi033.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.0.log 2024-09-21T10:10:13.715 INFO:teuthology.orchestra.run.smithi033.stderr:: No such file or directory 2024-09-21T10:10:13.715 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.1.log 2024-09-21T10:10:13.715 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-volume.log 2024-09-21T10:10:13.715 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.audit.log 2024-09-21T10:10:13.715 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.c.log 2024-09-21T10:10:13.715 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mds.a.log 2024-09-21T10:10:13.715 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.audit.log: /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log 2024-09-21T10:10:13.716 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mds.a.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.a.log 2024-09-21T10:10:13.716 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.2.log 2024-09-21T10:10:13.716 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.a.log: 91.0% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.cephadm.log.gz 2024-09-21T10:10:13.717 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.5.log: 90.5% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:10:13.718 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:10:13.718 INFO:teuthology.orchestra.run.smithi105.stderr:gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mgr.x.log 2024-09-21T10:10:13.718 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.b.log 2024-09-21T10:10:13.718 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:10:13.720 INFO:teuthology.orchestra.run.smithi033.stderr: 91.7% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mds.a.log.gz 2024-09-21T10:10:13.721 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mgr.y.log 2024-09-21T10:10:13.722 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.b.log: 85.8% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:10:13.724 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.2.log: 94.1%gzip -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.audit.log.gz 2024-09-21T10:10:13.724 INFO:teuthology.orchestra.run.smithi033.stderr: -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.log 2024-09-21T10:10:13.724 INFO:teuthology.orchestra.run.smithi033.stderr:gzip -5 --verbose -- /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.3.log 2024-09-21T10:10:13.731 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mgr.y.log: /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-21T10:10:13.731 INFO:teuthology.orchestra.run.smithi033.stderr: 92.1% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph.log.gz 2024-09-21T10:10:13.733 INFO:teuthology.orchestra.run.smithi105.stderr: 91.1% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:10:13.736 INFO:teuthology.orchestra.run.smithi033.stderr:/var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:10:13.737 INFO:teuthology.orchestra.run.smithi033.stderr: 91.1% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-volume.log.gz 2024-09-21T10:10:13.763 INFO:teuthology.orchestra.run.smithi105.stderr:/var/log/ceph/cephadm.log: 92.0% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-21T10:10:13.765 INFO:teuthology.orchestra.run.smithi105.stderr: 89.8% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mgr.x.log.gz 2024-09-21T10:10:14.176 INFO:teuthology.orchestra.run.smithi033.stderr: 89.2% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mgr.y.log.gz 2024-09-21T10:10:14.462 INFO:teuthology.orchestra.run.smithi105.stderr: 92.1% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.b.log.gz 2024-09-21T10:10:14.484 INFO:teuthology.orchestra.run.smithi033.stderr: 91.9% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.c.log.gz 2024-09-21T10:10:15.326 INFO:teuthology.orchestra.run.smithi033.stderr: 90.8% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-mon.a.log.gz 2024-09-21T10:10:18.182 INFO:teuthology.orchestra.run.smithi105.stderr: 94.4% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.6.log.gz 2024-09-21T10:10:18.687 INFO:teuthology.orchestra.run.smithi105.stderr: 94.4% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.7.log.gz 2024-09-21T10:10:18.713 INFO:teuthology.orchestra.run.smithi105.stderr: 94.1% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.4.log.gz 2024-09-21T10:10:19.352 INFO:teuthology.orchestra.run.smithi033.stderr: 93.8% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.2.log.gz 2024-09-21T10:10:19.465 INFO:teuthology.orchestra.run.smithi033.stderr: 94.0% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.0.log.gz 2024-09-21T10:10:19.917 INFO:teuthology.orchestra.run.smithi105.stderr: 94.0% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.5.log.gz 2024-09-21T10:10:19.919 INFO:teuthology.orchestra.run.smithi105.stderr: 2024-09-21T10:10:19.919 INFO:teuthology.orchestra.run.smithi105.stderr:real 0m6.220s 2024-09-21T10:10:19.920 INFO:teuthology.orchestra.run.smithi105.stderr:user 0m20.705s 2024-09-21T10:10:19.920 INFO:teuthology.orchestra.run.smithi105.stderr:sys 0m0.791s 2024-09-21T10:10:21.487 INFO:teuthology.orchestra.run.smithi033.stderr: 94.5% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.3.log.gz 2024-09-21T10:10:22.550 INFO:teuthology.orchestra.run.smithi033.stderr: 94.3% -- replaced with /var/log/ceph/a29e8460-77fd-11ef-baf6-efdc52797490/ceph-osd.1.log.gz 2024-09-21T10:10:22.552 INFO:teuthology.orchestra.run.smithi033.stderr: 2024-09-21T10:10:22.552 INFO:teuthology.orchestra.run.smithi033.stderr:real 0m8.853s 2024-09-21T10:10:22.552 INFO:teuthology.orchestra.run.smithi033.stderr:user 0m29.873s 2024-09-21T10:10:22.552 INFO:teuthology.orchestra.run.smithi033.stderr:sys 0m0.964s 2024-09-21T10:10:22.553 INFO:tasks.cephadm:Archiving logs... 2024-09-21T10:10:22.553 DEBUG:teuthology.misc:Transferring archived files from smithi033:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658/remote/smithi033/log 2024-09-21T10:10:22.554 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:10:26.417 DEBUG:teuthology.misc:Transferring archived files from smithi105:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658/remote/smithi105/log 2024-09-21T10:10:26.418 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-21T10:10:29.199 INFO:tasks.cephadm:Removing cluster... 2024-09-21T10:10:29.199 DEBUG:teuthology.orchestra.run.smithi033:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a29e8460-77fd-11ef-baf6-efdc52797490 --force 2024-09-21T10:10:29.727 DEBUG:teuthology.orchestra.run.smithi105:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a29e8460-77fd-11ef-baf6-efdc52797490 --force 2024-09-21T10:10:30.222 INFO:tasks.cephadm:Removing cephadm ... 2024-09-21T10:10:30.223 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:10:30.240 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-21T10:10:30.255 INFO:tasks.cephadm:Teardown complete 2024-09-21T10:10:30.255 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-21T10:10:30.265 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-21T10:10:30.266 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T10:10:30.283 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-21T10:10:30.369 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-21T10:10:30.379 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-21T10:10:30.380 DEBUG:teuthology.orchestra.run.smithi033:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:10:30.382 DEBUG:teuthology.orchestra.run.smithi105:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-21T10:10:30.396 INFO:teuthology.orchestra.run.smithi033.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:10:30.396 INFO:teuthology.orchestra.run.smithi105.stderr:bash: line 1: ntpq: command not found 2024-09-21T10:10:30.399 INFO:teuthology.orchestra.run.smithi033.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:10:30.400 INFO:teuthology.orchestra.run.smithi033.stdout:=============================================================================== 2024-09-21T10:10:30.400 INFO:teuthology.orchestra.run.smithi033.stdout:^* hv01.front.sepia.ceph.com 3 6 377 37 -14us[ -17us] +/- 43ms 2024-09-21T10:10:30.400 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 35 +192us[ +192us] +/- 51ms 2024-09-21T10:10:30.400 INFO:teuthology.orchestra.run.smithi033.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 37 -255us[ -255us] +/- 74ms 2024-09-21T10:10:30.400 INFO:teuthology.orchestra.run.smithi033.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:10:30.401 INFO:teuthology.orchestra.run.smithi105.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-21T10:10:30.402 INFO:teuthology.orchestra.run.smithi105.stdout:=============================================================================== 2024-09-21T10:10:30.402 INFO:teuthology.orchestra.run.smithi105.stdout:^* hv01.front.sepia.ceph.com 3 6 377 37 +15us[ +13us] +/- 43ms 2024-09-21T10:10:30.402 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 34 +179us[ +179us] +/- 51ms 2024-09-21T10:10:30.402 INFO:teuthology.orchestra.run.smithi105.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 38 -258us[ -259us] +/- 74ms 2024-09-21T10:10:30.402 INFO:teuthology.orchestra.run.smithi105.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-09-21T10:10:30.403 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-21T10:10:30.413 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-21T10:10:30.413 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-21T10:10:30.423 DEBUG:teuthology.orchestra.run.smithi033:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:10:30.533 DEBUG:teuthology.orchestra.run.smithi105:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-21T10:10:30.611 DEBUG:teuthology.orchestra.run.smithi033:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:10:30.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.642 DEBUG:teuthology.orchestra.run.smithi105:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-21T10:10:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:30.670 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-21T10:10:30.686 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-21T10:10:30.706 INFO:teuthology.task.internal:Duration was 2079.386347 seconds 2024-09-21T10:10:30.706 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-21T10:10:30.716 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-21T10:10:30.716 DEBUG:teuthology.orchestra.run.smithi033:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:10:30.719 DEBUG:teuthology.orchestra.run.smithi105:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-21T10:10:30.756 INFO:teuthology.orchestra.run.smithi033.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:10:30.757 INFO:teuthology.orchestra.run.smithi105.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-21T10:10:31.068 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-21T10:10:31.068 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi033.front.sepia.ceph.com 2024-09-21T10:10:31.069 DEBUG:teuthology.orchestra.run.smithi033:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:10:31.094 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi105.front.sepia.ceph.com 2024-09-21T10:10:31.094 DEBUG:teuthology.orchestra.run.smithi105:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-21T10:10:31.131 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-21T10:10:31.131 DEBUG:teuthology.orchestra.run.smithi033:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:10:31.137 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-21T10:10:31.197 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-21T10:10:31.198 DEBUG:teuthology.orchestra.run.smithi033:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:10:31.205 DEBUG:teuthology.orchestra.run.smithi105:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-21T10:10:31.675 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-21T10:10:31.691 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-21T10:10:31.692 DEBUG:teuthology.orchestra.run.smithi033:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:10:31.717 DEBUG:teuthology.orchestra.run.smithi105:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-21T10:10:31.742 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-21T10:10:31.752 DEBUG:teuthology.orchestra.run.smithi033:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:10:31.759 DEBUG:teuthology.orchestra.run.smithi105:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:10:31.781 INFO:teuthology.orchestra.run.smithi033.stdout:kernel.core_pattern = core 2024-09-21T10:10:31.805 INFO:teuthology.orchestra.run.smithi105.stdout:kernel.core_pattern = core 2024-09-21T10:10:31.845 DEBUG:teuthology.orchestra.run.smithi033:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:10:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.861 DEBUG:teuthology.orchestra.run.smithi105:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-21T10:10:31.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:31.901 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-21T10:10:31.912 INFO:teuthology.task.internal:Transferring archived files... 2024-09-21T10:10:31.913 DEBUG:teuthology.misc:Transferring archived files from smithi033:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658/remote/smithi033 2024-09-21T10:10:31.913 DEBUG:teuthology.orchestra.run.smithi033:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:10:31.960 DEBUG:teuthology.misc:Transferring archived files from smithi105:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-21_03:08:26-upgrade-main-distro-default-smithi/7913658/remote/smithi105 2024-09-21T10:10:31.960 DEBUG:teuthology.orchestra.run.smithi105:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-21T10:10:31.998 INFO:teuthology.task.internal:Removing archive directory... 2024-09-21T10:10:31.999 DEBUG:teuthology.orchestra.run.smithi033:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:10:32.001 DEBUG:teuthology.orchestra.run.smithi105:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-21T10:10:32.047 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-21T10:10:32.058 INFO:teuthology.task.internal:Not uploading archives. 2024-09-21T10:10:32.058 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-21T10:10:32.069 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-21T10:10:32.069 DEBUG:teuthology.orchestra.run.smithi033:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:10:32.072 DEBUG:teuthology.orchestra.run.smithi105:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-21T10:10:32.084 INFO:teuthology.orchestra.run.smithi033.stdout: 265324 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:10 /home/ubuntu/cephtest 2024-09-21T10:10:32.085 INFO:teuthology.orchestra.run.smithi033.stdout: 424627 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:04 /home/ubuntu/cephtest/mnt.0 2024-09-21T10:10:32.085 INFO:teuthology.orchestra.run.smithi033.stdout: 424715 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 21 10:08 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-21T10:10:32.085 INFO:teuthology.orchestra.run.smithi033.stdout: 426369 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:08 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-21T10:10:32.086 INFO:teuthology.orchestra.run.smithi033.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-21T10:10:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-21T10:10:32.089 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi033 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T10:10:32.096 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-21T10:10:32.107 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-21T10:10:32.119 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi033 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-21T10:10:32.123 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}} duration: 2079.3863472938538 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi033 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=6b75b9c430ac492ca49a3a1425b6cf0b status: fail success: false 2024-09-21T10:10:32.123 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-21T10:10:32.130 INFO:teuthology.orchestra.run.smithi105.stdout: 265336 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 21 10:10 /home/ubuntu/cephtest 2024-09-21T10:10:32.205 INFO:teuthology.run:FAIL